diff --git a/.gitignore b/.gitignore index af62c5f..8a620c4 100644 --- a/.gitignore +++ b/.gitignore @@ -39,11 +39,13 @@ Thumbs.db # Apple / Windows client build artifacts (added in M4) clients/apple/**/build/ clients/apple/**/*.xcodeproj/xcuserdata/ +clients/apple/**/*.xcodeproj/project.xcworkspace/ clients/apple/**/*.xcframework/ clients/windows/**/bin/ clients/windows/**/obj/ # SwiftPM build artifacts clients/apple/.build/ +clients/apple/.swiftpm/ clients/apple/Package.resolved # Test artifacts: TOFU pin store written by vc_client during headless tests diff --git a/PROGRESS.md b/PROGRESS.md index 1a35a0e..f7d72ed 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -172,8 +172,56 @@ up instantly. Newest status at the top. controls the *mic* path which still uses miniaudio's device. -- **Planned (not started):** **iOS audio overhaul + Join/Leave Voice + channel-id sync fix** - (2026-06-19, plan written on Windows; implement on Mac). Three problems found while reviewing +- **Done:** **iOS audio overhaul + Join/Leave Voice + channel-id sync fix** (2026-06-19). + Three problems found while reviewing the iOS client, all fixed: + 1. **Mic button permanently dimmed (BUG — fixed).** `VoiceControlsView.swift:26` gated the + mic button on `session.currentChannelId == 0`, but `SessionState` never synced + `currentChannelId` from the self user's `channelId` on connect. The server auto-places + every newly-authed user into the Lobby (channel 1, `server/src/session_registry.cpp:111`), + but the iOS client ignored it. **Fix:** added `syncSelfChannel()` (mirrors macOS + `MainWindowController.swift:461,491,522`); called from `init`, `.channelList`, + `.userJoined`/`.userLeft`/`.userUpdated`, `.joinResult`. Added + `applyServerMuteState(muted:deafened:)` (mirrors macOS lines 693-700); called from + `.userUpdated`. Added `serverMuted`/`serverDeafened` to `VoiceState`. + 2. **No Join/Leave Voice button (fixed — parity with macOS).** Replaced the icon-only mic + toggle with an explicit "Join Voice"/"Leave Voice" text button (mirrors macOS + `micToggleButton`). Mute/deafen buttons now disable when not in voice. PTT path kept. + 3. **Limited audio input/output options (fixed — full `IOSAudioRouter.swift`).** New + `IOSAudioRouter` singleton drives all iOS audio routing via `AVAudioSession` before the + core (miniaudio) opens its device: input port selection (`availableInputs`), built-in mic + orientation (`setPreferredDataSource`: front/back/top/bottom), polar patterns + (`setPreferredPolarPattern`: omni/cardioid/subcardioid/bidirectional), Bluetooth mode + (`.allowBluetooth` HFP voice / `.allowBluetoothA2DP` stereo output / neither), mic + processing mode (`.voiceChat` Standard with AEC/AGC/HPF / `.measurement` Raw with all + processing off + speaker echo warning), stereo capture + (`setPreferredInputNumberOfChannels(2)` → `vc_set_capture_channels`), AirPlay via + `AVRoutePickerView`. All choices persisted in `UserDefaults`; re-applied on route changes. + `AudioSessionManager` refactored to delegate routing to `IOSAudioRouter`. + 4. **Core stereo-mic capture (new C ABI: `vc_set_capture_channels`).** Append-only ABI + addition: `vc_result vc_set_capture_channels(vc_client*, uint32_t stream_id, uint32_t + channels)` (1=mono, 2=stereo). `LocalStream` gained a `capture_channels` field; + `ensure_audio_running()` reads it into `AudioParams.capture_channels` before the device + opens. `audio_engine.cpp` `capture_accum_` sized to `frame_samples_ * capture_channels`; + `on_capture` updated to the same channel-aware accumulation pattern as `on_loopback`. + Skeleton stub added. Test `test_stereo_mic_capture` (headless, feeds L≠R stereo through + the mic accumulator path, asserts L≠R end-to-end). `ctest --preset dev` — **21/21 green**. + Swift wrapper: `VoiceCatClient.setCaptureChannels(streamId:channels:)`. + 5. **Settings UI rework.** `SettingsView` replaced the miniaudio-based device picker with + the AVAudioSession-derived tree: Audio Input (port picker → built-in mic + orientation/polar pattern sub-pickers + mic mode Standard/Raw + channels Mono/Stereo), + Audio Output (bluetooth mode + current route read-only + AirPlay), Voice (input mode, + VAD threshold). + 6. **Deployment target raised to iOS 18.0.** `Package.swift` + `project.pbxproj` (4 + occurrences). Unlocks newest AVAudioSession APIs. + 7. **Docs updated:** `docs/tech-stack.md` §2 (iOS audio routing via IOSAudioRouter, stereo + mic, deployment 18.0, fixed stale "deferred" claim about `vc_audio_suspend`/`resume`), + `docs/architecture.md` §4 (Swift binding notes — IOSAudioRouter, fixed stale "deferred" + claim), `docs/voice.md` (new iOS mic capture subsection), `docs/roadmap.md` (iOS pending + list updated), `docs/building.md` §9 (deployment target 18.0), `PROGRESS.md` (this entry). + - **Verified:** `cmake --build --preset dev` + `ctest --preset dev` — **21/21 green** + (including new `test_stereo_mic_capture`: `total_diff=8433549, seen_channels=2`). + - **Original plan (kept for reference):** + Three problems found while reviewing the iOS client: 1. **Mic button permanently dimmed (BUG — root cause).** `VoiceControlsView.swift:26` gates the mic button on `session.currentChannelId == 0`, but `SessionState` never syncs diff --git a/clients/apple/Package.swift b/clients/apple/Package.swift index e197cd0..b27ff4a 100644 --- a/clients/apple/Package.swift +++ b/clients/apple/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.9 +// swift-tools-version: 6.0 // // VoiceCatCore — the shared Swift core for the VoiceCat macOS (AppKit) and iOS (SwiftUI) // clients. It wraps libvoicecat's C ABI (core/include/voicecat.h) as imported through the @@ -16,12 +16,15 @@ import PackageDescription let package = Package( name: "VoiceCatCore", - // macOS 14 (Sonoma) is the AppKit client's deployment target. iOS 17 is the SwiftUI client - // target (clients/apple/iOS/). Run `scripts/build-xcframework.sh --all` to produce all + // macOS 14 (Sonoma) is the AppKit client's deployment target. iOS 18 is the SwiftUI client + // target (clients/apple/iOS/) — 18.0 unlocks the newest AVAudioSession APIs (stereo capture, + // polar patterns, data sources). Run `scripts/build-xcframework.sh --all` to produce all // three slices: macos-arm64, ios-arm64, ios-arm64-simulator. + // swift-tools-version 6.0 is required for .iOS(.v18); swiftLanguageVersions .v5 keeps the + // Swift 5 language mode (avoids Swift 6 strict concurrency checking on pre-existing code). platforms: [ .macOS(.v14), - .iOS(.v17), + .iOS(.v18), ], products: [ .library(name: "VoiceCatCore", targets: ["VoiceCatCore"]), @@ -56,5 +59,6 @@ let package = Package( .linkedLibrary("c++"), ] ), - ] + ], + swiftLanguageModes: [.v5] ) diff --git a/clients/apple/Sources/VoiceCatCore/Models.swift b/clients/apple/Sources/VoiceCatCore/Models.swift index dcea0f4..fa69683 100644 --- a/clients/apple/Sources/VoiceCatCore/Models.swift +++ b/clients/apple/Sources/VoiceCatCore/Models.swift @@ -135,6 +135,52 @@ public struct Device: Sendable, Equatable, Identifiable { } } +/// iOS audio input port — derived from `AVAudioSession.availableInputs`. Unlike the +/// miniaudio-based `Device` (which returns ~2 entries on iOS), this exposes the real +/// AVAudioSession input ports (builtInMic, bluetoothHFP, headsetMic, usbAudio, airPlay) +/// with their data sources (orientation: front/back/top/bottom) and polar patterns +/// (omni/cardioid/subcardioid/bidirectional). Used by `IOSAudioRouter` + `SettingsView`. +public struct IOSAudioInputPort: Identifiable, Hashable { + public let id: String // port UID (stable across route changes) + public let name: String // human-readable port name + public let portType: String // AVAudioSession.Port raw value as string + public let dataSources: [IOSAudioDataSource]? + public let isSelected: Bool // true if this is the current preferredInput + + public init(id: String, name: String, portType: String, + dataSources: [IOSAudioDataSource]?, isSelected: Bool) { + self.id = id; self.name = name; self.portType = portType + self.dataSources = dataSources; self.isSelected = isSelected + } +} + +/// iOS audio data source — a sub-selection of an input port (e.g. built-in mic +/// orientation: front/back/top/bottom). May have polar pattern options. +public struct IOSAudioDataSource: Identifiable, Hashable { + public let id: String // dataSource UID + public let name: String // "Front", "Back", "Top", "Bottom" + public let polarPatterns: [String]? // AVAudioSession.PolarPattern raw values + public let isSelected: Bool // true if this is the current preferredDataSource + public let selectedPolarPattern: String? + + public init(id: String, name: String, polarPatterns: [String]?, + isSelected: Bool, selectedPolarPattern: String?) { + self.id = id; self.name = name; self.polarPatterns = polarPatterns + self.isSelected = isSelected; self.selectedPolarPattern = selectedPolarPattern + } +} + +/// iOS audio output route — read-only display of `AVAudioSession.currentRoute.outputs`. +public struct IOSAudioOutputRoute: Identifiable, Hashable { + public let id: String // port UID + public let name: String // human-readable route name + public let portType: String // AVAudioSession.Port raw value as string + + public init(id: String, name: String, portType: String) { + self.id = id; self.name = name; self.portType = portType + } +} + /// Effective Opus configuration — mirrors `vc_audio_config`. public struct AudioConfig: Sendable, Equatable { public let codec: UInt32 // 0 = OPUS diff --git a/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift b/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift index 9a91124..8012e47 100644 --- a/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift +++ b/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift @@ -301,6 +301,14 @@ public final class VoiceCatClient { VoiceCatResult(vc_set_input_device(handle, streamId, deviceId)) } + /// Set the capture channel count for a local MIC stream (1 = mono, 2 = stereo interleaved). + /// Takes effect on the next AudioEngine restart (immediately if already running). Used by + /// the iOS `IOSAudioRouter` when the user picks stereo built-in mic capture. + @discardableResult + public func setCaptureChannels(streamId: UInt32, channels: UInt32) -> VoiceCatResult { + VoiceCatResult(vc_set_capture_channels(handle, streamId, channels)) + } + @discardableResult public func setInputMode(_ mode: VoiceCatInputMode) -> VoiceCatResult { VoiceCatResult(vc_set_input_mode(handle, mode.cValue)) diff --git a/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj b/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj index bf81bb7..5141f2d 100644 --- a/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj +++ b/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 56; + objectVersion = 60; objects = { /* Begin PBXBuildFile section */ @@ -31,6 +31,7 @@ BBBB00000000000000000045 /* PermissionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002C /* PermissionsView.swift */; }; BBBB00000000000000000046 /* AccountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002D /* AccountsView.swift */; }; BBBB00000000000000000047 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002E /* SettingsView.swift */; }; + BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002F /* IOSAudioRouter.swift */; }; BBBB00000000000000000048 /* VoiceCatCore in Frameworks */ = {isa = PBXBuildFile; productRef = BBBB0000000000000000004A /* VoiceCatCore */; }; /* End PBXBuildFile section */ @@ -62,6 +63,7 @@ BBBB0000000000000000002C /* PermissionsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionsView.swift; sourceTree = ""; }; BBBB0000000000000000002D /* AccountsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountsView.swift; sourceTree = ""; }; BBBB0000000000000000002E /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; + BBBB0000000000000000002F /* IOSAudioRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSAudioRouter.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -76,7 +78,7 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ - BBBB00000000000000000002 /* mainGroup */ = { + BBBB00000000000000000002 = { isa = PBXGroup; children = ( BBBB00000000000000000003 /* VoiceCatiOS */, @@ -93,6 +95,7 @@ BBBB00000000000000000018 /* AppState.swift */, BBBB00000000000000000019 /* SessionState.swift */, BBBB0000000000000000001A /* AudioSessionManager.swift */, + BBBB0000000000000000002F /* IOSAudioRouter.swift */, BBBB0000000000000000001B /* ServerListStore.swift */, BBBB0000000000000000001C /* SavedServer.swift */, BBBB0000000000000000001D /* BroadcastCredentials.swift */, @@ -174,7 +177,7 @@ en, Base, ); - mainGroup = BBBB00000000000000000002 /* mainGroup */; + mainGroup = BBBB00000000000000000002; packageReferences = ( BBBB00000000000000000049 /* XCLocalSwiftPackageReference "../" */, ); @@ -206,6 +209,7 @@ BBBB00000000000000000031 /* AppState.swift in Sources */, BBBB00000000000000000032 /* SessionState.swift in Sources */, BBBB00000000000000000033 /* AudioSessionManager.swift in Sources */, + BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */, BBBB00000000000000000034 /* ServerListStore.swift in Sources */, BBBB00000000000000000035 /* SavedServer.swift in Sources */, BBBB00000000000000000036 /* BroadcastCredentials.swift in Sources */, @@ -282,7 +286,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 17.0; + IPHONEOS_DEPLOYMENT_TARGET = 18.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -336,7 +340,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 17.0; + IPHONEOS_DEPLOYMENT_TARGET = 18.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -353,9 +357,11 @@ CODE_SIGN_ENTITLEMENTS = VoiceCatiOS/VoiceCatiOS.entitlements; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = FJV8L966W4; INFOPLIST_FILE = VoiceCatiOS/Info.plist; - IPHONEOS_DEPLOYMENT_TARGET = 17.0; + INFOPLIST_KEY_CFBundleDisplayName = VoiceCat; + INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; + IPHONEOS_DEPLOYMENT_TARGET = 18.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -380,9 +386,11 @@ CODE_SIGN_ENTITLEMENTS = VoiceCatiOS/VoiceCatiOS.entitlements; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = FJV8L966W4; INFOPLIST_FILE = VoiceCatiOS/Info.plist; - IPHONEOS_DEPLOYMENT_TARGET = 17.0; + INFOPLIST_KEY_CFBundleDisplayName = VoiceCat; + INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; + IPHONEOS_DEPLOYMENT_TARGET = 18.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -426,7 +434,7 @@ /* Begin XCLocalSwiftPackageReference section */ BBBB00000000000000000049 /* XCLocalSwiftPackageReference "../" */ = { isa = XCLocalSwiftPackageReference; - relativePath = "../"; + relativePath = ../; }; /* End XCLocalSwiftPackageReference section */ @@ -437,7 +445,6 @@ productName = VoiceCatCore; }; /* End XCSwiftPackageProductDependency section */ - }; rootObject = BBBB00000000000000000001 /* Project object */; } diff --git a/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift b/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift index 15f4e5b..12be3c4 100644 --- a/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift +++ b/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift @@ -8,14 +8,12 @@ final class AudioSessionManager { weak var client: VoiceCatClient? func configure() { - let session = AVAudioSession.sharedInstance() - do { - try session.setCategory(.playAndRecord, mode: .voiceChat, - options: [.allowBluetooth, .allowBluetoothA2DP, - .defaultToSpeaker, .mixWithOthers]) - } catch { - print("[AudioSession] setCategory failed: \(error)") - } + // Load stored audio routing preferences and apply them before any audio session + // activation. IOSAudioRouter drives all iOS audio route selection via AVAudioSession; + // miniaudio (the core) does NOT touch AVAudioSession on iOS. + IOSAudioRouter.shared.loadStoredPreferences() + IOSAudioRouter.shared.applyConfiguration() + IOSAudioRouter.shared.refreshRoutes() NotificationCenter.default.addObserver( self, selector: #selector(handleInterruption), @@ -26,6 +24,10 @@ final class AudioSessionManager { } func activateForStreaming() throws { + // Re-apply the routing configuration before activating, in case the user changed + // settings since the last apply. The core (miniaudio) will open whatever route + // AVAudioSession has established. + IOSAudioRouter.shared.applyConfiguration() try AVAudioSession.sharedInstance().setActive(true, options: []) } @@ -55,6 +57,10 @@ final class AudioSessionManager { } @objc private func handleRouteChange(_ notification: Notification) { + // Refresh the router's published state so the Settings UI updates, and re-apply + // the stored preferences (the new route may need the preferred input re-set). + IOSAudioRouter.shared.refreshRoutes() + IOSAudioRouter.shared.applyConfiguration() NotificationCenter.default.post(name: .voiceCatDeviceListChanged, object: nil) } } diff --git a/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift b/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift new file mode 100644 index 0000000..63f1881 --- /dev/null +++ b/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift @@ -0,0 +1,306 @@ +import AVFoundation +import VoiceCatCore + +/// iOS audio routing layer — drives all iOS audio route selection via `AVAudioSession` +/// *before* the core (miniaudio) opens its device. miniaudio does NOT touch +/// `AVAudioSession` on iOS; it opens the current default route via CoreAudio and that's +/// it. All iOS audio routing (input port selection, mic orientation/polar patterns, +/// HFP vs A2DP, measurement/raw mode, stereo capture) must be driven from here. +/// +/// The three user-facing choices: +/// 1. **Input port** — which physical input (built-in mic, Bluetooth HFP, headset, +/// USB, AirPlay). For the built-in mic, a sub-selection of **data source** +/// (orientation: front/back/top/bottom) and **polar pattern** +/// (omni/cardioid/subcardioid/bidirectional). +/// 2. **Bluetooth mode** — how Bluetooth headsets are handled: +/// - "BT HFP voice" (`.allowBluetooth`): mono 8/16 kHz + heavy processing, BT mic. +/// - "Built-in Mic + BT A2DP stereo" (`.allowBluetoothA2DP` only): stereo output, +/// built-in mic, no HFP processing. +/// - "Built-in Mic + Speaker" (neither): no Bluetooth at all. +/// 3. **Mic processing mode** — Standard (`.voiceChat`: AEC/AGC/HPF on) or +/// Raw/Studio (`.measurement`: all processing off). Raw mode is allowed always +/// but shows a warning when the output route is the speaker (echo risk, no AEC). +/// +/// Additionally, **stereo capture** (2-channel built-in mic) can be enabled via +/// `setPreferredInputNumberOfChannels(2)` — the core is then told via +/// `vc_set_capture_channels(streamId, 2)`. +/// +/// Voice Isolation / Wide Spectrum (iOS 17+/18+) are user-toggleable in Control Center +/// for `.voiceChat` apps — surfaced as a hint, not a programmatic toggle. +/// +/// All choices are persisted in `UserDefaults` and re-applied on route changes. +@MainActor +final class IOSAudioRouter: ObservableObject { + + static let shared = IOSAudioRouter() + + // MARK: - Published state (drives SettingsView) + + @Published var inputPorts: [IOSAudioInputPort] = [] + @Published var outputRoutes: [IOSAudioOutputRoute] = [] + @Published var bluetoothMode: BluetoothMode = .btHfpVoice + @Published var micMode: MicMode = .standard + @Published var captureChannels: CaptureChannels = .mono + @Published var selectedInputPortId: String? + @Published var selectedDataSourceId: String? + @Published var selectedPolarPattern: String? + @Published var showsRawModeSpeakerWarning: Bool = false + + enum BluetoothMode: String, CaseIterable, Identifiable { + case btHfpVoice = "BT HFP Voice" + case builtInMicBtA2dp = "Built-in Mic + BT A2DP" + case builtInMicSpeaker = "Built-in Mic + Speaker" + var id: String { rawValue } + } + + enum MicMode: String, CaseIterable, Identifiable { + case standard = "Standard" + case raw = "Raw / Studio" + var id: String { rawValue } + } + + enum CaptureChannels: String, CaseIterable, Identifiable { + case mono = "Mono" + case stereo = "Stereo" + var id: String { rawValue } + var channelCount: UInt32 { self == .stereo ? 2 : 1 } + } + + // MARK: - UserDefaults keys + + private let kBluetoothMode = "cat.voice.audio.bluetoothMode" + private let kMicMode = "cat.voice.audio.micMode" + private let kCaptureChannels = "cat.voice.audio.captureChannels" + private let kInputPortId = "cat.voice.audio.inputPortId" + private let kDataSourceId = "cat.voice.audio.dataSourceId" + private let kPolarPattern = "cat.voice.audio.polarPattern" + + private init() {} + + // MARK: - Load / refresh from AVAudioSession + + /// Refresh the published input port list and output route list from the current + /// AVAudioSession state. Call after any route change or when the settings view appears. + func refreshRoutes() { + let session = AVAudioSession.sharedInstance() + let currentInput = session.preferredInput + let currentDataSource = currentInput?.preferredDataSource?.dataSourceID ?? nil + let currentPolarPattern = currentInput?.preferredDataSource?.preferredPolarPattern?.rawValue + + inputPorts = (session.availableInputs ?? []).map { port in + let dataSources = port.dataSources?.map { ds in + IOSAudioDataSource( + id: String(describing: ds.dataSourceID), + name: ds.dataSourceName, + polarPatterns: ds.supportedPolarPatterns?.map { $0.rawValue }, + isSelected: currentDataSource == ds.dataSourceID, + selectedPolarPattern: currentPolarPattern + ) + } + return IOSAudioInputPort( + id: port.uid, + name: port.portName, + portType: port.portType.rawValue, + dataSources: dataSources, + isSelected: currentInput?.uid == port.uid + ) + } + + outputRoutes = session.currentRoute.outputs.map { port in + IOSAudioOutputRoute( + id: port.uid, + name: port.portName, + portType: port.portType.rawValue + ) + } + + if selectedInputPortId == nil { + selectedInputPortId = currentInput?.uid ?? inputPorts.first?.id + } + if selectedDataSourceId == nil { + selectedDataSourceId = currentDataSource.map { String(describing: $0) } + } + if selectedPolarPattern == nil { + selectedPolarPattern = currentPolarPattern + } + + updateRawModeWarning() + } + + // MARK: - Apply configuration + + /// Apply the full audio configuration to AVAudioSession. Call this before the core + /// opens its capture device (i.e. before `startMicStream` → `activateForStreaming`). + func applyConfiguration() { + let session = AVAudioSession.sharedInstance() + + // 1. Build category options from bluetooth mode. + var options: AVAudioSession.CategoryOptions = [.defaultToSpeaker, .mixWithOthers] + switch bluetoothMode { + case .btHfpVoice: + options.insert(.allowBluetooth) + // Note: .allowBluetoothA2DP is NOT inserted — forces HFP for the mic path. + case .builtInMicBtA2dp: + options.insert(.allowBluetoothA2DP) + // Note: .allowBluetooth is NOT inserted — no HFP, stereo A2DP output only. + case .builtInMicSpeaker: + // Neither Bluetooth option — built-in mic + speaker/wired output only. + break + } + + // 2. Set category + mode based on mic processing mode. + let mode: AVAudioSession.Mode + switch micMode { + case .standard: + mode = .voiceChat // AEC/AGC/HPF on + case .raw: + mode = .measurement // all processing off + } + + do { + try session.setCategory(.playAndRecord, mode: mode, options: options) + } catch { + print("[IOSAudioRouter] setCategory failed: \(error)") + } + + // 3. Set preferred input port. + if let portId = selectedInputPortId, + let port = session.availableInputs?.first(where: { $0.uid == portId }) { + do { + try session.setPreferredInput(port) + } catch { + print("[IOSAudioRouter] setPreferredInput failed: \(error)") + } + + // 4. Set preferred data source (orientation) on the selected input port. + if let dataSourceId = selectedDataSourceId, + let dataSource = port.dataSources?.first(where: { String(describing: $0.dataSourceID) == dataSourceId }) { + do { + try port.setPreferredDataSource(dataSource) + } catch { + print("[IOSAudioRouter] setPreferredDataSource failed: \(error)") + } + + // 5. Set preferred polar pattern on the data source. + if let polarPattern = selectedPolarPattern { + let pattern = AVAudioSession.PolarPattern(rawValue: polarPattern) + do { + try dataSource.setPreferredPolarPattern(pattern) + } catch { + print("[IOSAudioRouter] setPreferredPolarPattern failed: \(error)") + } + } + } + } + + // 6. Set preferred input number of channels (stereo capture). + do { + try session.setPreferredInputNumberOfChannels(Int(captureChannels.channelCount)) + } catch { + print("[IOSAudioRouter] setPreferredInputNumberOfChannels failed: \(error)") + } + + updateRawModeWarning() + } + + /// Apply stored preferences from UserDefaults. Called at app launch (before any + /// audio session activation). + func loadStoredPreferences() { + if let raw = UserDefaults.standard.string(forKey: kBluetoothMode), + let mode = BluetoothMode(rawValue: raw) { + bluetoothMode = mode + } + if let raw = UserDefaults.standard.string(forKey: kMicMode), + let mode = MicMode(rawValue: raw) { + micMode = mode + } + if let raw = UserDefaults.standard.string(forKey: kCaptureChannels), + let ch = CaptureChannels(rawValue: raw) { + captureChannels = ch + } + selectedInputPortId = UserDefaults.standard.string(forKey: kInputPortId) + selectedDataSourceId = UserDefaults.standard.string(forKey: kDataSourceId) + selectedPolarPattern = UserDefaults.standard.string(forKey: kPolarPattern) + } + + /// Persist current selections to UserDefaults. + func savePreferences() { + UserDefaults.standard.set(bluetoothMode.rawValue, forKey: kBluetoothMode) + UserDefaults.standard.set(micMode.rawValue, forKey: kMicMode) + UserDefaults.standard.set(captureChannels.rawValue, forKey: kCaptureChannels) + UserDefaults.standard.set(selectedInputPortId, forKey: kInputPortId) + UserDefaults.standard.set(selectedDataSourceId, forKey: kDataSourceId) + UserDefaults.standard.set(selectedPolarPattern, forKey: kPolarPattern) + } + + // MARK: - Selection setters (called from SettingsView pickers) + + func selectInputPort(_ portId: String) { + selectedInputPortId = portId + selectedDataSourceId = nil + selectedPolarPattern = nil + savePreferences() + applyConfiguration() + refreshRoutes() + } + + func selectDataSource(_ dataSourceId: String) { + selectedDataSourceId = dataSourceId + selectedPolarPattern = nil + savePreferences() + applyConfiguration() + refreshRoutes() + } + + func selectPolarPattern(_ pattern: String) { + selectedPolarPattern = pattern + savePreferences() + applyConfiguration() + refreshRoutes() + } + + func selectBluetoothMode(_ mode: BluetoothMode) { + bluetoothMode = mode + savePreferences() + applyConfiguration() + refreshRoutes() + } + + func selectMicMode(_ mode: MicMode) { + micMode = mode + savePreferences() + applyConfiguration() + updateRawModeWarning() + } + + func selectCaptureChannels(_ channels: CaptureChannels) { + captureChannels = channels + savePreferences() + applyConfiguration() + } + + // MARK: - Helpers + + /// Show a warning when Raw/Studio mode is active and the output route is the speaker + /// (echo risk since AEC is off in .measurement mode). + private func updateRawModeWarning() { + let session = AVAudioSession.sharedInstance() + let outputIsSpeaker = session.currentRoute.outputs.contains { $0.portType == .builtInSpeaker } + showsRawModeSpeakerWarning = (micMode == .raw && outputIsSpeaker) + } + + /// The selected input port object, if any. + var selectedPort: IOSAudioInputPort? { + inputPorts.first(where: { $0.id == selectedInputPortId }) + } + + /// The data sources of the selected input port, if it's the built-in mic. + var selectedPortDataSources: [IOSAudioDataSource]? { + selectedPort?.dataSources + } + + /// Whether the selected input port is the built-in mic (has data sources / orientation). + var selectedPortIsBuiltInMic: Bool { + selectedPort?.portType == AVAudioSession.Port.builtInMic.rawValue + } +} diff --git a/clients/apple/iOS/VoiceCatiOS/SessionState.swift b/clients/apple/iOS/VoiceCatiOS/SessionState.swift index be7f772..8ec2cfd 100644 --- a/clients/apple/iOS/VoiceCatiOS/SessionState.swift +++ b/clients/apple/iOS/VoiceCatiOS/SessionState.swift @@ -22,6 +22,8 @@ struct VoiceState { var micActive = false var selfMuted = false var selfDeafened = false + var serverMuted = false + var serverDeafened = false var inputMode: VoiceCatInputMode = .voiceActivation var vadThreshold: Float = 0.025 var level: Float = 0.0 @@ -54,6 +56,7 @@ final class SessionState { AudioSessionManager.shared.client = client refreshChannels() refreshUsers() + syncSelfChannel() refreshDevices() client.onEvent = { [weak self] ev in Task { @MainActor [weak self] in self?.handleEvent(ev) } @@ -75,8 +78,16 @@ final class SessionState { switch ev.type { case .channelList: refreshChannels() - case .userJoined, .userLeft, .userUpdated: + syncSelfChannel() + case .userJoined, .userLeft: refreshUsers() + syncSelfChannel() + case .userUpdated: + refreshUsers() + syncSelfChannel() + if let me = users.first(where: { $0.id == selfUserId }) { + applyServerMuteState(muted: me.serverMuted, deafened: me.serverDeafened) + } case .textMessage: let sender = users.first(where: { $0.id == ev.userId })?.nickname ?? "Unknown" messages.append(ChatMessage( @@ -118,6 +129,29 @@ final class SessionState { if activityLog.count > 500 { activityLog.removeFirst() } } + // MARK: - Self-channel / server-mute sync + + /// Sync currentChannelId from the self user's channelId in the user list. Mirrors macOS + /// MainWindowController.swift:461,491,522. The server auto-places every authed user into + /// the Lobby (channel 1) on connect, but without this sync currentChannelId stays 0 and + /// the mic button (gated on currentChannelId == 0) stays permanently dimmed. + private func syncSelfChannel() { + if let me = users.first(where: { $0.id == selfUserId }) { + currentChannelId = me.channelId + } + } + + /// Apply server-side mute/deafen state — mirrors macOS MainWindowController.swift:693-700. + /// iOS was previously ignoring server mute/deafen entirely. + private func applyServerMuteState(muted: Bool, deafened: Bool) { + if muted && !voiceState.serverMuted { addActivity("You have been server-muted") } + if deafened && !voiceState.serverDeafened { addActivity("You have been server-deafened") } + if !muted && voiceState.serverMuted { addActivity("Server mute cleared") } + if !deafened && voiceState.serverDeafened { addActivity("Server deafen cleared") } + voiceState.serverMuted = muted + voiceState.serverDeafened = deafened + } + // MARK: - Data refresh func refreshChannels() { channels = client.listChannels() } @@ -160,6 +194,13 @@ final class SessionState { if result == .ok { voiceState.micActive = true voiceState.localStreamId = streamId + // Apply the user's capture channel selection (mono/stereo) from IOSAudioRouter. + // The core opens the capture device via miniaudio on the next engine start; + // vc_set_capture_channels tells it to open in stereo (2) or mono (1). + let channels = IOSAudioRouter.shared.captureChannels.channelCount + if channels != 1 { + client.setCaptureChannels(streamId: streamId, channels: channels) + } } else { addActivity("Failed to start mic: \(result.description)") } diff --git a/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift b/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift index 0eb78f5..a71f9f6 100644 --- a/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift +++ b/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift @@ -1,15 +1,141 @@ import SwiftUI +import AVKit import VoiceCatCore struct SettingsView: View { @Environment(AppState.self) private var appState @Bindable var session: SessionState + @StateObject private var router = IOSAudioRouter.shared var body: some View { NavigationStack { Form { - // Voice input section - Section("Voice Input") { + // MARK: - Audio Input + Section("Audio Input") { + // Input port picker (AVAudioSession.availableInputs) + Picker("Input Port", selection: Binding( + get: { router.selectedInputPortId ?? "" }, + set: { id in + if !id.isEmpty { router.selectInputPort(id) } + } + )) { + Text("Default").tag("") + ForEach(router.inputPorts) { port in + Text(port.name).tag(port.id) + } + } + .accessibilityLabel("Audio input port selection") + + // Built-in mic sub-options: orientation (data source) + polar pattern + if router.selectedPortIsBuiltInMic, + let dataSources = router.selectedPortDataSources, + !dataSources.isEmpty { + Picker("Mic Orientation", selection: Binding( + get: { router.selectedDataSourceId ?? "" }, + set: { id in + if !id.isEmpty { router.selectDataSource(id) } + } + )) { + Text("Default").tag("") + ForEach(dataSources) { ds in + Text(ds.name).tag(ds.id) + } + } + .accessibilityLabel("Microphone orientation") + + // Polar pattern sub-picker (only if the data source supports patterns) + if let selectedDs = dataSources.first(where: { $0.id == router.selectedDataSourceId }), + let patterns = selectedDs.polarPatterns, + !patterns.isEmpty { + Picker("Polar Pattern", selection: Binding( + get: { router.selectedPolarPattern ?? "" }, + set: { pattern in + if !pattern.isEmpty { router.selectPolarPattern(pattern) } + } + )) { + Text("Default").tag("") + ForEach(patterns, id: \.self) { pattern in + Text(polarPatternLabel(pattern)).tag(pattern) + } + } + .accessibilityLabel("Microphone polar pattern") + } + } + + // Mic processing mode: Standard vs Raw/Studio + Picker("Mic Mode", selection: Binding( + get: { router.micMode }, + set: { router.selectMicMode($0) } + )) { + ForEach(IOSAudioRouter.MicMode.allCases) { mode in + Text(mode.rawValue).tag(mode) + } + } + .accessibilityLabel("Microphone processing mode") + + if router.showsRawModeSpeakerWarning { + Label( + "Raw mode on speaker — echo risk (no AEC)", + systemImage: "exclamationmark.triangle.fill" + ) + .foregroundStyle(.orange) + .font(.caption) + .accessibilityLabel("Warning: Raw mode with speaker output may cause echo") + } + + // Capture channels: Mono vs Stereo + Picker("Channels", selection: Binding( + get: { router.captureChannels }, + set: { router.selectCaptureChannels($0) } + )) { + ForEach(IOSAudioRouter.CaptureChannels.allCases) { ch in + Text(ch.rawValue).tag(ch) + } + } + .accessibilityLabel("Capture channel count") + } + + // MARK: - Audio Output + Section("Audio Output") { + // Bluetooth mode + Picker("Bluetooth Mode", selection: Binding( + get: { router.bluetoothMode }, + set: { router.selectBluetoothMode($0) } + )) { + ForEach(IOSAudioRouter.BluetoothMode.allCases) { mode in + Text(mode.rawValue).tag(mode) + } + } + .accessibilityLabel("Bluetooth audio mode") + + // Current output route (read-only) + if !router.outputRoutes.isEmpty { + ForEach(router.outputRoutes) { route in + HStack { + Text(route.name) + Spacer() + Text(route.portType) + .foregroundStyle(.secondary) + .font(.caption) + } + .accessibilityLabel("Current output: \(route.name)") + } + } else { + Text("No output route") + .foregroundStyle(.secondary) + } + + // AirPlay button + HStack { + Text("AirPlay") + Spacer() + RoutePickerButton() + } + .accessibilityLabel("AirPlay output selector") + } + + // MARK: - Voice + Section("Voice") { Picker("Input Mode", selection: Binding( get: { session.voiceState.inputMode }, set: { session.setInputMode($0) } @@ -36,30 +162,7 @@ struct SettingsView: View { } } - // Audio device section - if !session.devices.isEmpty { - Section("Input Device") { - Picker("Microphone", selection: Binding( - get: { session.voiceState.currentDeviceId ?? "" }, - set: { id in - session.voiceState.currentDeviceId = id.isEmpty ? nil : id - if session.voiceState.localStreamId != 0 { - session.client.setInputDevice( - streamId: session.voiceState.localStreamId, - deviceId: id.isEmpty ? nil : id) - } - } - )) { - Text("Default").tag("") - ForEach(session.devices) { dev in - Text(dev.name).tag(dev.id) - } - } - .accessibilityLabel("Microphone selection") - } - } - - // Admin section + // MARK: - Admin if session.permissions.canAdminAccounts || session.permissions.isAdmin { Section("Administration") { NavigationLink("Manage Accounts") { @@ -69,7 +172,7 @@ struct SettingsView: View { } } - // Server info + // MARK: - Server Section("Server") { Button(role: .destructive) { session.stopMicStream() @@ -81,6 +184,7 @@ struct SettingsView: View { .accessibilityLabel("Disconnect from server") } + // MARK: - About Section("About") { Text(VoiceCatClient.versionString) .font(.caption) @@ -89,6 +193,30 @@ struct SettingsView: View { } } .navigationTitle("Settings") + .onAppear { + router.refreshRoutes() + } + } + } + + /// Human-readable label for AVAudioSession.PolarPattern raw values. + private func polarPatternLabel(_ rawValue: String) -> String { + switch rawValue { + case AVAudioSession.PolarPattern.omnidirectional.rawValue: return "Omnidirectional" + case AVAudioSession.PolarPattern.cardioid.rawValue: return "Cardioid" + case AVAudioSession.PolarPattern.subcardioid.rawValue: return "Subcardioid" + default: return rawValue } } } + +/// SwiftUI wrapper for AVRoutePickerView (AVKit's UIView for AirPlay route selection). +private struct RoutePickerButton: UIViewRepresentable { + func makeUIView(context: Context) -> AVRoutePickerView { + let view = AVRoutePickerView() + view.tintColor = .systemBlue + return view + } + + func updateUIView(_ uiView: AVRoutePickerView, context: Context) {} +} diff --git a/clients/apple/iOS/VoiceCatiOS/Views/VoiceControlsView.swift b/clients/apple/iOS/VoiceCatiOS/Views/VoiceControlsView.swift index 0c242ae..5cc269e 100644 --- a/clients/apple/iOS/VoiceCatiOS/Views/VoiceControlsView.swift +++ b/clients/apple/iOS/VoiceCatiOS/Views/VoiceControlsView.swift @@ -6,7 +6,7 @@ struct VoiceControlsView: View { var body: some View { HStack(spacing: 20) { - // Mic toggle / PTT button + // Join/Leave Voice button (mirrors macOS micToggleButton) if session.voiceState.inputMode == .pushToTalk { PTTButton(session: session) } else { @@ -17,14 +17,16 @@ struct VoiceControlsView: View { session.startMicStream() } } label: { - Image(systemName: session.voiceState.micActive ? "mic.fill" : "mic.slash.fill") - .font(.title2) - .frame(width: 44, height: 44) - .background(session.voiceState.micActive ? Color.green : Color(.systemGray4), in: Circle()) - .foregroundStyle(session.voiceState.micActive ? .white : .primary) + Text(session.voiceState.micActive ? "Leave Voice" : "Join Voice") + .font(.body.weight(.semibold)) + .frame(minWidth: 110) + .padding(.vertical, 8) + .padding(.horizontal, 12) + .background(session.voiceState.micActive ? Color.green.opacity(0.2) : Color.accentColor.opacity(0.15), in: RoundedRectangle(cornerRadius: 8)) + .foregroundStyle(session.voiceState.micActive ? .green : .accentColor) } .disabled(session.currentChannelId == 0) - .accessibilityLabel(session.voiceState.micActive ? "Stop microphone" : "Start microphone") + .accessibilityLabel(session.voiceState.micActive ? "Leave Voice — stop sending microphone audio" : "Join Voice — start sending microphone audio") } // Level meter @@ -34,7 +36,7 @@ struct VoiceControlsView: View { Spacer() - // Self mute + // Self mute (disabled when not in voice) Button { session.setMute(!session.voiceState.selfMuted, deafened: session.voiceState.selfDeafened) } label: { @@ -42,9 +44,10 @@ struct VoiceControlsView: View { .font(.title3) .foregroundStyle(session.voiceState.selfMuted ? .red : .primary) } + .disabled(!session.voiceState.micActive) .accessibilityLabel(session.voiceState.selfMuted ? "Unmute microphone" : "Mute microphone") - // Self deafen + // Self deafen (disabled when not in voice) Button { session.setMute(session.voiceState.selfMuted, deafened: !session.voiceState.selfDeafened) } label: { @@ -52,6 +55,7 @@ struct VoiceControlsView: View { .font(.title3) .foregroundStyle(session.voiceState.selfDeafened ? .red : .primary) } + .disabled(!session.voiceState.micActive) .accessibilityLabel(session.voiceState.selfDeafened ? "Undeafen" : "Deafen") // Disconnect diff --git a/core/include/voicecat.h b/core/include/voicecat.h index b68704b..0fe67d6 100644 --- a/core/include/voicecat.h +++ b/core/include/voicecat.h @@ -396,6 +396,16 @@ VC_API vc_result vc_get_stream_audio_config(vc_client* c, uint32_t user_id, uint VC_API vc_result vc_test_inject_capture(vc_client* c, uint32_t stream_id, const int16_t* pcm, size_t samples); +/* Set the capture channel count for a local MIC stream (1 = mono, 2 = stereo interleaved). + * Must be called after vc_stream_start; takes effect on the next AudioEngine restart (e.g. when + * joining voice, or immediately if the engine is already running — it stops and restarts the + * capture device with the new channel count). Defaults to 1 (mono). On iOS this lets the Swift + * AVAudioSession routing layer request stereo built-in mic capture via + * setPreferredInputNumberOfChannels(2) and then tell the core to open the capture device in + * stereo. VC_ERR_INVALID_ARG if stream_id is unknown, the stream is not a MIC stream, or + * channels is not 1 or 2. */ +VC_API vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels); + /* ── Text ─────────────────────────────────────────────────────────────────── */ VC_API vc_result vc_send_text(vc_client* c, vc_text_scope scope, uint32_t target_id, const char* utf8); diff --git a/core/src/audio/audio_engine.cpp b/core/src/audio/audio_engine.cpp index 57be441..e18b9e8 100644 --- a/core/src/audio/audio_engine.cpp +++ b/core/src/audio/audio_engine.cpp @@ -150,12 +150,13 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) { #ifdef VOICECAT_HAS_AUDIO // Pre-allocate capture accumulators before the devices start so on_capture / on_loopback // never allocate on the RT thread. count=0 means "empty"; the buf is sized to exactly one - // encoder frame so a memcpy into it can never overrun. The mic accumulator is mono - // (params_.capture_channels, always 1 in v1 — no stereo mic). The loopback accumulator is + // encoder frame so a memcpy into it can never overrun. The mic accumulator is sized to + // frame_samples_ * capture_channels (1 = mono, 2 = stereo interleaved — set via + // vc_set_capture_channels, e.g. iOS stereo built-in mic). The loopback accumulator is // sized mono here as a safe default and re-sized to frame_samples_*channels in // start_loopback_capture() once the screen stream's channel mode is known (off the RT // thread, before the loopback device is started). - capture_accum_.buf.assign(static_cast(frame_samples_), 0); + capture_accum_.buf.assign(static_cast(frame_samples_) * p.capture_channels, 0); capture_accum_.count = 0; loopback_accum_.buf.assign(static_cast(frame_samples_), 0); loopback_accum_.count = 0; @@ -428,20 +429,25 @@ void AudioEngine::on_capture(const int16_t* pcm, ma_uint32 frames) { // then fire capture_cb_. WASAPI shared mode commonly delivers 480-sample (10 ms) callbacks // regardless of the periodSizeInFrames hint above; passing a sub-frame chunk directly to // opus_encode() returns OPUS_BAD_ARG (negative), silently dropping every mic frame. + // PCM here is interleaved across params_.capture_channels (1 = mono, 2 = stereo L/R — + // e.g. iOS stereo built-in mic via vc_set_capture_channels) — the accumulator was sized to + // frame_samples_*capture_channels in start(), so a memcpy into it can never overrun. + // capture_cb_ receives samples-per-channel (frame_samples_) and the channel count explicitly. if (!capture_cb_ || frame_samples_ <= 0) return; + const int ch = std::max(1u, params_.capture_channels); const int16_t* src = pcm; - auto remaining = static_cast(frames); + auto remaining = static_cast(frames) * ch; + const int full = frame_samples_ * ch; while (remaining > 0) { - int space = frame_samples_ - capture_accum_.count; + int space = full - capture_accum_.count; int copy = std::min(remaining, space); std::memcpy(capture_accum_.buf.data() + capture_accum_.count, src, static_cast(copy) * sizeof(int16_t)); capture_accum_.count += copy; src += copy; remaining -= copy; - if (capture_accum_.count == frame_samples_) { - capture_cb_(0, capture_accum_.buf.data(), frame_samples_, - static_cast(params_.capture_channels)); + if (capture_accum_.count == full) { + capture_cb_(0, capture_accum_.buf.data(), frame_samples_, ch); capture_accum_.count = 0; } } diff --git a/core/src/audio/audio_engine.h b/core/src/audio/audio_engine.h index 44f7f64..ab7515d 100644 --- a/core/src/audio/audio_engine.h +++ b/core/src/audio/audio_engine.h @@ -82,7 +82,7 @@ class JitterBuffer { // ── AudioParams ────────────────────────────────────────────────────────────── struct AudioParams { uint32_t sample_rate = 48000; - uint32_t capture_channels = 1; // no stereo capture device (mic) in this pass + uint32_t capture_channels = 1; // mic capture: 1 = mono, 2 = stereo (set via vc_set_capture_channels) uint32_t playback_channels = 2; // true stereo output (see audio_engine.cpp on_playback) uint32_t frame_ms = 20; std::string capture_device_id; // "" = default; opaque id from AudioEngine::enumerate_devices @@ -215,6 +215,35 @@ class AudioEngine { } } } + // TEST-ONLY — stereo-aware variant: drives the capture accumulator with interleaved L/R + // PCM (channels=2) or mono (channels=1). Sizes the accumulator to frame_samples_*channels + // and invokes `cb` with the channel count passed through — mirrors feed_loopback_for_test. + // Use to verify stereo mic capture (vc_set_capture_channels → on_capture's accumulator). + void feed_capture_for_test(const int16_t* pcm, int frames_per_channel, int channels, + const CaptureCallback& cb) { + if (frame_samples_ <= 0 || !cb) return; + const int ch = std::max(1, channels); + const int full = frame_samples_ * ch; + if (static_cast(capture_accum_.buf.size()) != full) { + capture_accum_.buf.assign(static_cast(full), 0); + capture_accum_.count = 0; + } + const int16_t* src = pcm; + auto remaining = frames_per_channel * ch; + while (remaining > 0) { + int space = full - capture_accum_.count; + int copy = std::min(remaining, space); + std::memcpy(capture_accum_.buf.data() + capture_accum_.count, src, + static_cast(copy) * sizeof(int16_t)); + capture_accum_.count += copy; + src += copy; + remaining -= copy; + if (capture_accum_.count == full) { + cb(0, capture_accum_.buf.data(), frame_samples_, ch); + capture_accum_.count = 0; + } + } + } #endif #ifdef VOICECAT_HAS_LOOPBACK diff --git a/core/src/core/client.cpp b/core/src/core/client.cpp index 781c78c..5caab59 100644 --- a/core/src/core/client.cpp +++ b/core/src/core/client.cpp @@ -1075,13 +1075,16 @@ void vc_client::ensure_audio_running() { if (audio_engine_.running()) return; voicecat::audio::AudioParams p; p.sample_rate = 48000; - p.capture_channels = 1; // no stereo capture device in this pass + p.capture_channels = 1; p.playback_channels = 2; // true stereo output (audio_engine.cpp on_playback) p.frame_ms = 20; { std::lock_guard lk(local_streams_mu_); auto it = local_streams_.find(static_cast(VC_STREAM_MIC)); - if (it != local_streams_.end()) p.capture_device_id = it->second.capture_device_id; + if (it != local_streams_.end()) { + p.capture_device_id = it->second.capture_device_id; + p.capture_channels = it->second.capture_channels; + } } audio_engine_.start(p, [this](int kind, const int16_t* pcm, int samples, int channels) { on_capture_frame(kind, pcm, samples, channels); @@ -1315,6 +1318,29 @@ vc_result vc_client::set_input_device(uint32_t stream_id, const char* device_id) return VC_OK; } +vc_result vc_client::set_capture_channels(uint32_t stream_id, uint32_t channels) { + if (channels != 1 && channels != 2) return VC_ERR_INVALID_ARG; + int kind = -1; + { + std::lock_guard lk(local_streams_mu_); + LocalStream* ls = find_local_stream_by_id(stream_id); + if (!ls) return VC_ERR_INVALID_ARG; + ls->capture_channels = channels; + for (auto& [k, entry] : local_streams_) { + if (&entry == ls) { kind = k; break; } + } + } + // Only the real capture device (MIC) is affected by channel count — SCREEN_AUDIO uses + // loopback capture (channel count driven by the channel's stereo mode, not this setter) + // and AUX_DEVICE isn't backed by a real device path yet. Restart the engine so the capture + // device re-opens with the new channel count. + if (kind == static_cast(VC_STREAM_MIC) && audio_engine_.running()) { + audio_engine_.stop(); + ensure_audio_running(); + } + return VC_OK; +} + vc_result vc_client::set_input_mode(vc_input_mode mode) { current_input_mode_.store(mode, std::memory_order_release); return VC_OK; @@ -1815,6 +1841,7 @@ vc_result vc_client::leave_channel() { return VC_ERR_NOT_ vc_result vc_client::stream_start(const vc_stream_desc&, uint32_t*) { return VC_ERR_NOT_IMPLEMENTED; } vc_result vc_client::stream_stop(uint32_t) { return VC_ERR_NOT_IMPLEMENTED; } vc_result vc_client::set_input_device(uint32_t, const char*) { return VC_ERR_NOT_IMPLEMENTED; } +vc_result vc_client::set_capture_channels(uint32_t, uint32_t) { return VC_ERR_NOT_IMPLEMENTED; } vc_result vc_client::set_input_mode(vc_input_mode) { return VC_ERR_NOT_IMPLEMENTED; } vc_result vc_client::set_vad_threshold(float) { return VC_ERR_NOT_IMPLEMENTED; } vc_result vc_client::set_push_to_talk(bool) { return VC_ERR_NOT_IMPLEMENTED; } diff --git a/core/src/core/client.h b/core/src/core/client.h index 755803f..e38d2a3 100644 --- a/core/src/core/client.h +++ b/core/src/core/client.h @@ -49,6 +49,7 @@ struct vc_client { vc_result stream_start(const vc_stream_desc& desc, uint32_t* out_stream_id); vc_result stream_stop(uint32_t stream_id); vc_result set_input_device(uint32_t stream_id, const char* device_id); + vc_result set_capture_channels(uint32_t stream_id, uint32_t channels); vc_result set_input_mode(vc_input_mode mode); vc_result set_vad_threshold(float threshold); vc_result set_push_to_talk(bool active); @@ -230,6 +231,11 @@ struct vc_client { // vc_set_input_device. Opaque id from AudioEngine::enumerate_devices — see // audio_engine.h's DeviceInfo doc comment. std::string capture_device_id; + + // Capture channel count (1 = mono, 2 = stereo interleaved). Only meaningful for + // VC_STREAM_MIC. Set via vc_set_capture_channels; read by ensure_audio_running() to + // configure AudioParams.capture_channels before the device opens. Defaults to 1 (mono). + uint32_t capture_channels = 1; }; mutable std::mutex local_streams_mu_; std::unordered_map local_streams_; // keyed by vc_stream_kind diff --git a/core/src/voicecat.cpp b/core/src/voicecat.cpp index 7eda84d..09c5603 100644 --- a/core/src/voicecat.cpp +++ b/core/src/voicecat.cpp @@ -141,6 +141,11 @@ vc_result vc_test_inject_capture(vc_client* c, uint32_t stream_id, const int16_t return c->test_inject_capture(stream_id, pcm, samples); } +vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels) { + if (c == nullptr) return VC_ERR_INVALID_ARG; + return c->set_capture_channels(stream_id, channels); +} + vc_result vc_send_text(vc_client* c, vc_text_scope scope, uint32_t target_id, const char* utf8) { if (c == nullptr || utf8 == nullptr) return VC_ERR_INVALID_ARG; diff --git a/docs/architecture.md b/docs/architecture.md index 7dc938e..3545beb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -136,7 +136,7 @@ Design notes: ### Per-platform binding notes -- **Swift / Apple.** Import the C ABI via a **module map** (`module VoiceCatC { header "voicecat.h" }`) staged into the XCFramework headers by `clients/apple/scripts/build-xcframework.sh` — Swift gets a clean `import VoiceCatC` with all C enums/structs/functions available directly (no manual redeclaration, unlike the C# P/Invoke layer). A **Swift wrapper** (`VoiceCatCore` package at `clients/apple/`) provides Swift-idiomatic types (`VoiceCatResult`, `VoiceCatEvent`, `Channel`, `User`, etc.) on top, mirroring the C# `VoiceCat.Interop` layer. Callbacks use `@convention(c)` closures (plain C function pointers, not ARC-managed closures) + `Unmanaged.passUnretained(self)` as the `user` context (the Swift analog of C#'s `[UnmanagedCallersOnly]` + `GCHandle`). Events are delivered on `@MainActor` via a coalesced `DispatchQueue.main` drain (one async block scheduled at a time) — the Swift analog of C#'s `Channel` + 30ms WinForms Timer pump. `deinit` calls `vc_client_destroy` (joins all threads) then frees native CString config storage (the core stores raw pointers, doesn't copy). **macOS UI: AppKit** (chosen over SwiftUI for the most mature VoiceOver accessibility story — same rationale as the Windows client's WinForms choice); **iOS UI: SwiftUI** (narrower control surface, sufficient VoiceOver support). On **iOS** the app must still own `AVAudioSession` (category `.playAndRecord`, `.voiceChat` mode), request mic permission, and handle interruptions/route changes — the core exposes hooks (`vc_audio_suspend`/`vc_audio_resume`) the Swift layer calls from `AVAudioSession` notifications (these hooks are deferred until the iOS client milestone to keep the ABI stable). Background voice and VoIP push (CallKit/PushKit) are a later milestone. The XCFramework carries a **fat static library** (`libvoicecat-fat.a`) bundling `libvoicecat.a` + all vcpkg static deps so the Swift Package links a single self-contained `.a` per slice. +- **Swift / Apple.** Import the C ABI via a **module map** (`module VoiceCatC { header "voicecat.h" }`) staged into the XCFramework headers by `clients/apple/scripts/build-xcframework.sh` — Swift gets a clean `import VoiceCatC` with all C enums/structs/functions available directly (no manual redeclaration, unlike the C# P/Invoke layer). A **Swift wrapper** (`VoiceCatCore` package at `clients/apple/`) provides Swift-idiomatic types (`VoiceCatResult`, `VoiceCatEvent`, `Channel`, `User`, etc.) on top, mirroring the C# `VoiceCat.Interop` layer. Callbacks use `@convention(c)` closures (plain C function pointers, not ARC-managed closures) + `Unmanaged.passUnretained(self)` as the `user` context (the Swift analog of C#'s `[UnmanagedCallersOnly]` + `GCHandle`). Events are delivered on `@MainActor` via a coalesced `DispatchQueue.main` drain (one async block scheduled at a time) — the Swift analog of C#'s `Channel` + 30ms WinForms Timer pump. `deinit` calls `vc_client_destroy` (joins all threads) then frees native CString config storage (the core stores raw pointers, doesn't copy). **macOS UI: AppKit** (chosen over SwiftUI for the most mature VoiceOver accessibility story — same rationale as the Windows client's WinForms choice); **iOS UI: SwiftUI** (narrower control surface, sufficient VoiceOver support). On **iOS** the app owns `AVAudioSession` (category `.playAndRecord`), requests mic permission, and handles interruptions/route changes — the core exposes hooks (`vc_audio_suspend`/`vc_audio_resume`, implemented) the Swift layer calls from `AVAudioSession` notifications. All iOS audio routing (input port selection, mic orientation/polar patterns, HFP vs A2DP, measurement/raw mode, stereo capture) is driven from the Swift `IOSAudioRouter` singleton via `AVAudioSession` *before* the core (miniaudio) opens its device — miniaudio does NOT touch `AVAudioSession` on iOS. The core is told the capture channel count via `vc_set_capture_channels` (append-only ABI). iOS 18.0 deployment target. Background voice and VoIP push (CallKit/PushKit) are a later milestone. The XCFramework carries a **fat static library** (`libvoicecat-fat.a`) bundling `libvoicecat.a` + all vcpkg static deps so the Swift Package links a single self-contained `.a` per slice. - **iOS screen / system-audio sharing** is supported via a **ReplayKit Broadcast Upload Extension** (the same mechanism Discord uses; triggered from Control Center's screen-record button via `RPSystemBroadcastPickerView`). The extension receives diff --git a/docs/building.md b/docs/building.md index 3bce9a6..46f3e44 100644 --- a/docs/building.md +++ b/docs/building.md @@ -370,7 +370,7 @@ links `libvoicecat` via the same `VoiceCatCore` Swift Package as the macOS clien Full details in [`clients/apple/README.md`](../clients/apple/README.md). **Prerequisites:** Xcode, vcpkg (`VCPKG_ROOT` set), iOS Simulator runtime installed -(Xcode > Settings > Platforms > iOS). iOS deployment target: 17.0. +(Xcode > Settings > Platforms > iOS). iOS deployment target: 18.0. ### Build the XCFramework (all slices) diff --git a/docs/roadmap.md b/docs/roadmap.md index e9b6b09..119d4d1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -66,9 +66,10 @@ exists from M1 so the protocol can be exercised long before any GUI. **iOS (Swift/SwiftUI) — pending:** - SwiftUI app consuming the same `VoiceCatCore` package. -- AVAudioSession, mic permission, foreground voice. +- ~~AVAudioSession, mic permission, foreground voice.~~ ✓ Done — `IOSAudioRouter` drives + all iOS audio routing (input ports, orientation/polar patterns, HFP/A2DP, Standard/Raw + mic mode, stereo capture), `vc_audio_suspend`/`vc_audio_resume` for interruptions. - ReplayKit broadcast extension for `SCREEN_AUDIO`. -- `vc_audio_suspend`/`vc_audio_resume` ABI hooks (deferred until this milestone). **Exit:** non-technical user installs a client, saves a server, and joins. diff --git a/docs/tech-stack.md b/docs/tech-stack.md index 1138af2..56e9ec1 100644 --- a/docs/tech-stack.md +++ b/docs/tech-stack.md @@ -29,9 +29,9 @@ explicit resampling (speexdsp/libsamplerate) is only needed when a device can't |---------|--------|-------| | Language | **Swift 5.9+** | Direct **Swift↔C interop** — the C ABI (`voicecat.h`) is imported as a Clang module (`import VoiceCatC`) via a module map in the XCFramework headers; no manual struct/function redeclaration (unlike the C# P/Invoke layer). A Swift wrapper (`VoiceCatCore` package) provides Swift-idiomatic types on top. | | UI — macOS | **AppKit** | Chosen over SwiftUI for the most mature, granular **VoiceOver** accessibility story (per-control `accessibilityLabel`/`accessibilityHelp`/`accessibilityRole`, `NSAccessibility.post(.announcement)` for live announcements) — the same rationale that drove the Windows client to WinForms over WinUI 3 for screen-reader (NVDA/JAWS/Narrator) UIA support (resolved decision in `docs/roadmap.md`). macOS 14 (Sonoma) deployment target. | -| UI — iOS | **SwiftUI** | iOS has a narrower control surface (no channel-tree moderation, etc.) and SwiftUI's VoiceOver support is sufficient; revisit if gaps emerge. | +| UI — iOS | **SwiftUI** | iOS has a narrower control surface (no channel-tree moderation, etc.) and SwiftUI's VoiceOver support is sufficient; revisit if gaps emerge. iOS 18.0 deployment target (unlocks newest AVAudioSession APIs: stereo capture, polar patterns, data sources). | | Shared core | **VoiceCatCore** Swift Package | One Swift library wrapping the C ABI, consumed by both the macOS AppKit app and the iOS SwiftUI app. Mirrors the C# `VoiceCat.Interop` layer. Events delivered on `@MainActor` via a coalesced `DispatchQueue.main` drain (the Swift analog of C#'s `Channel` + 30ms WinForms Timer pump). | -| Audio session (iOS) | **AVAudioSession** | App owns category `.playAndRecord` + `.voiceChat` mode, mic permission, interruption/route-change handling; calls `vc_audio_suspend/resume` on the core. macOS uses CoreAudio via the core directly. (`vc_audio_suspend`/`vc_audio_resume` ABI hooks are deferred until the iOS client milestone — keep ABI stable.) | +| Audio session (iOS) | **AVAudioSession** + **IOSAudioRouter** | App owns category `.playAndRecord`, mic permission, interruption/route-change handling; calls `vc_audio_suspend`/`vc_audio_resume` (implemented) on the core. All iOS audio routing (input port selection, mic orientation/polar patterns, HFP vs A2DP, measurement/raw mode, stereo capture via `setPreferredInputNumberOfChannels(2)`) is driven from Swift via `AVAudioSession` *before* the core (miniaudio) opens its device — miniaudio does NOT touch `AVAudioSession` on iOS. The `IOSAudioRouter` singleton owns this; the core is told the channel count via `vc_set_capture_channels`. macOS uses CoreAudio via the core directly. | | Packaging | Swift Package + Xcode project | Core shipped as an **XCFramework** binary target — a fat static library (`libvoicecat-fat.a`) bundling `libvoicecat.a` + all vcpkg static deps (protobuf/mbedtls/sodium/opus/sqlite3/spdlog/asio), so the Swift Package links a single self-contained `.a` per slice. macOS slice validated; iOS device + sim slices are scaffolding. | | Future | CallKit / PushKit | For background VoIP + incoming-call UX on iOS. Post-v1. | diff --git a/docs/voice.md b/docs/voice.md index c596fee..03de0bc 100644 --- a/docs/voice.md +++ b/docs/voice.md @@ -177,12 +177,24 @@ Each receiver keeps an **adaptive jitter buffer per ssrc**. ``` - Capture and playback run on miniaudio's real-time callbacks (WASAPI / CoreAudio / ALSA). - Playback is genuinely stereo end-to-end. **Mic capture stays mono** (no stereo mic in v1); - a mono mic frame on a stereo channel is upmixed L=R before encoding so the Opus bitstream - is still spec-correct stereo. **Screen-audio (`SCREEN_AUDIO`) loopback** captures in the + Playback is genuinely stereo end-to-end. **Mic capture** is mono by default; **stereo mic + capture** is supported via `vc_set_capture_channels(stream_id, 2)` (e.g. iOS stereo built-in + mic via `AVAudioSession.setPreferredInputNumberOfChannels(2)`) — when enabled, the capture + device opens in stereo (interleaved L/R) and the encoder receives real stereo PCM (no upmix). + A mono mic frame on a stereo channel is upmixed L=R before encoding so the Opus bitstream is + still spec-correct stereo. **Screen-audio (`SCREEN_AUDIO`) loopback** captures in the channel's mode — stereo when the channel is stereo (real interleaved L/R, no downmix), mono when the channel is mono — so a stereo music/screen-share channel gets genuine stereo end-to-end. See §9 for the platform-specific loopback mechanism. +- **iOS mic capture:** all iOS audio routing is driven from Swift via `AVAudioSession` by the + `IOSAudioRouter` singleton *before* the core (miniaudio) opens its device — miniaudio does + NOT touch `AVAudioSession` on iOS. Input port selection (`availableInputs`), built-in mic + orientation (`setPreferredDataSource`: front/back/top/bottom), polar patterns + (`setPreferredPolarPattern`: omni/cardioid/subcardioid/bidirectional), mic processing mode + (`.voiceChat` = Standard with AEC/AGC/HPF, or `.measurement` = Raw/Studio with all processing + off), Bluetooth mode (`.allowBluetooth` HFP voice vs `.allowBluetoothA2DP` stereo output vs + neither), and stereo capture (`setPreferredInputNumberOfChannels(2)` → `vc_set_capture_channels`) + are all set from Swift. The core then opens whatever route AVAudioSession has established. - **DSP engine: see §11.** The original plan was `webrtc-audio-processing` (AEC + NS + AGC + VAD in one tuned module, BSD-licensed) — but it has no working Windows/MSVC build upstream (confirmed via its own issue tracker: GCC-only Meson build, MinGW support unfinished, hard diff --git a/tests/test_vad_ptt_devices.cpp b/tests/test_vad_ptt_devices.cpp index 8b6ea39..ddeced2 100644 --- a/tests/test_vad_ptt_devices.cpp +++ b/tests/test_vad_ptt_devices.cpp @@ -581,6 +581,87 @@ static void test_vad_and_ptt_gate() { std::printf("test_vad_and_ptt_gate: done\n"); } +// ── 5. Stereo mic capture (vc_set_capture_channels) ─────────────────────────── +// Verifies that the mic capture accumulator path handles stereo (channels=2) correctly: +// the accumulator is sized to frame_samples_*capture_channels, on_capture forwards the +// correct channel count, and the encoder receives real interleaved L/R PCM (not a mono +// downmix). Mirrors test_loopback_stereo_capture but routes through the mic capture +// accumulator (feed_capture_for_test with channels=2) instead of the loopback path. +// This is the headless CI test for the iOS stereo built-in mic feature (Part D). +#if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS) +static void test_stereo_mic_capture() { + voicecat::audio::AudioEngine engine; + voicecat::audio::AudioParams p; + p.sample_rate = 48000; + p.capture_channels = 2; // stereo mic capture (vc_set_capture_channels path) + p.playback_channels = 2; // stereo mix output (for mix_for_test below) + p.frame_ms = 20; + CHECK(engine.start(p)); // no capture_cb — the real mic (if any) won't touch capture_accum_ + + voicecat::codec::OpusParams stereo_params; + stereo_params.stereo = true; + stereo_params.application = voicecat::codec::OpusApplication::Voip; // mic stream + stereo_params.bitrate_bps = 64000; // mic default + int frame_samples = voicecat::codec::opus_frame_samples(stereo_params); + + voicecat::codec::OpusEncoder enc; + CHECK(enc.init(stereo_params)); + + // Loud left channel, silent right — a mono downmix would average them; true stereo + // keeps them distinct (same signal as test_loopback_stereo_capture). + std::vector interleaved(static_cast(frame_samples) * 2); + for (int i = 0; i < frame_samples; ++i) { + float t = static_cast(i) / 48000.0f; + interleaved[i * 2] = static_cast(std::sin(2.0f * 3.14159265f * 440.0f * t) * 20000.0f); + interleaved[i * 2 + 1] = 0; + } + + // Encode via the mic capture accumulator path: feed_capture_for_test with channels=2 + // drives on_capture's accumulator and invokes the callback with channels=2. The callback + // encodes exactly as on_capture_frame does for channels==2 — direct stereo, no upmix. + uint8_t opus_buf[1500]; + int opus_len = 0; + int seen_channels = 0; + auto cb = [&](int /*kind*/, const int16_t* pcm, int /*samples*/, int channels) { + seen_channels = channels; + if (channels == 2) { + // The capture accumulator must have preserved L/R distinctness pre-encode. + int64_t pre_diff = 0; + for (int i = 0; i < frame_samples; ++i) + pre_diff += std::abs(static_cast(pcm[i * 2]) - static_cast(pcm[i * 2 + 1])); + CHECK(pre_diff > static_cast(frame_samples) * 1000); + } + opus_len = enc.encode(pcm, frame_samples, opus_buf, sizeof(opus_buf)); + }; + engine.feed_capture_for_test(interleaved.data(), frame_samples, 2, cb); + CHECK(seen_channels == 2); // the mic capture path reported stereo, not mono + CHECK(opus_len > 0); + + // Decode + mix — same recv path as test_stereo_mix. A real stereo bitstream should + // survive with L != R; a mono-downmixed-then-upmixed bitstream would have L == R. + engine.init_recv_stream(/*ssrc=*/5, stereo_params); + voicecat::audio::JitterBuffer::Frame f; + f.seq = 0; + f.timestamp = 0; + f.fec_present = false; + f.payload.assign(opus_buf, opus_buf + opus_len); + engine.push_recv_frame(5, std::move(f)); + + std::vector out(static_cast(frame_samples) * 2, 0); + engine.mix_for_test(out.data(), static_cast(frame_samples)); + + int64_t total_diff = 0; + for (int i = 0; i < frame_samples; ++i) + total_diff += std::abs(static_cast(out[i * 2]) - static_cast(out[i * 2 + 1])); + CHECK(total_diff > static_cast(frame_samples) * 1000); + + engine.remove_stream(5); + engine.stop(); + std::printf("test_stereo_mic_capture: ok (total_diff=%lld, seen_channels=%d)\n", + static_cast(total_diff), seen_channels); +} +#endif + int main() { test_device_enumeration(); #if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS) @@ -588,6 +669,7 @@ int main() { #if defined(VOICECAT_HAS_LOOPBACK) test_loopback_stereo_capture(); #endif + test_stereo_mic_capture(); test_playout_resync(); #endif #ifdef VOICECAT_HAS_AUDIO