The iOS audio path was a hybrid: Voice-Chat-class presets ran a native VPIO AVAudioEngine (core external) while Stereo/Studio/A2DP presets ran the core's miniaudio devices. Nearly every "no input / no output / both" bug lived in the seam between the two paths — the lingering miniaudio capture unit fighting VPIO, the audioRestart ordering dance, the route-change "glitching" loop, stereo<->mono stickiness, and "can't hear anyone". Switching presets/routes mid-call routinely dropped a direction. Drive ALL iOS audio through one AVAudioEngine with the core fully external at all times: setExternalPlayback(1) once at connect, every MIC stream external_feed=1, mic via vc_stream_feed_pcm, playback via vc_set_mixed_output_sink (drained by an always-on AVAudioSourceNode so remote audio plays before joining voice). VPIO + AGC toggle per preset. Every preset/route/interruption change funnels through one deterministic Swift-only reconfigure (stop -> apply session config -> rebuild -> start) — no second path to hand off to, so a change can't drop a direction. - IOSVoiceProcessingEngine.swift -> IOSAudioEngine: always-on source-node playback, conditional mic tap, VPIO/AGC; one rebuild() backing startListening/stop/startMic/stopMic/reconfigure/setCaptureChannels. - IOSAudioRouter: 7 presets -> 4 (Voice Chat / Stereo Mic / Mono Mic / Advanced); persisted voiceProcessingEnabled + agcEnabled; setters call IOSAudioEngine.reconfigure() instead of audioRestart/reconcileVoicePath. - AudioSessionManager slimmed; SessionState mic lifecycle collapsed; AppState wires external playback + listening at connect, stop at disconnect; SettingsView shows 4 presets + Advanced VPIO/AGC toggles. No core/ABI/test changes — relies on the already-shipped external API (test_external_pcm, test_external_playback). xcodebuild iOS device Debug BUILD SUCCEEDED. Updates docs/voice.md §8 and PROGRESS.md.
410 lines
16 KiB
Swift
410 lines
16 KiB
Swift
import Foundation
|
|
import AVFoundation
|
|
import VoiceCatCore
|
|
|
|
// MARK: - Helper types
|
|
|
|
struct ChatMessage: Identifiable {
|
|
let id = UUID()
|
|
let timestamp: Date
|
|
let senderName: String
|
|
let text: String
|
|
let scope: VoiceCatTextScope
|
|
}
|
|
|
|
struct ActivityEntry: Identifiable {
|
|
let id = UUID()
|
|
let timestamp: Date
|
|
let text: String
|
|
}
|
|
|
|
struct VoiceState {
|
|
var micActive = false
|
|
var selfMuted = false
|
|
var selfDeafened = false
|
|
var serverMuted = false
|
|
var serverDeafened = false
|
|
var inputMode: VoiceCatInputMode = .voiceActivation
|
|
var vadThreshold: Float = 0.025
|
|
var level: Float = 0.0
|
|
var currentDeviceId: String?
|
|
var localStreamId: UInt32 = 0
|
|
var screenSharing = false
|
|
var screenStreamId: UInt32 = 0
|
|
}
|
|
|
|
// MARK: - SessionState
|
|
|
|
@Observable
|
|
@MainActor
|
|
final class SessionState {
|
|
let client: VoiceCatClient
|
|
let selfUserId: UInt32
|
|
|
|
var channels: [Channel] = []
|
|
var users: [User] = []
|
|
var currentChannelId: UInt32 = 0
|
|
var messages: [ChatMessage] = []
|
|
var activityLog: [ActivityEntry] = []
|
|
var voiceState = VoiceState()
|
|
var permissions: Permissions
|
|
var accounts: [Account] = []
|
|
var devices: [Device] = []
|
|
|
|
/// Host side of iOS screen-audio sharing — drains the broadcast extension's App Group ring
|
|
/// and feeds the SCREEN_AUDIO stream this session owns. See BroadcastAudioPump.
|
|
private let broadcastPump = BroadcastAudioPump()
|
|
|
|
init(client: VoiceCatClient, selfUserId: UInt32, permissions: Permissions) {
|
|
self.client = client
|
|
self.selfUserId = selfUserId
|
|
self.permissions = permissions
|
|
refreshChannels()
|
|
refreshUsers()
|
|
syncSelfChannel()
|
|
refreshDevices()
|
|
client.onEvent = { [weak self] ev in
|
|
Task { @MainActor [weak self] in self?.handleEvent(ev) }
|
|
}
|
|
client.onLevel = { [weak self] _, rms in
|
|
Task { @MainActor [weak self] in self?.voiceState.level = rms }
|
|
}
|
|
broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() }
|
|
broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() }
|
|
broadcastPump.start()
|
|
}
|
|
|
|
deinit {
|
|
broadcastPump.stop()
|
|
}
|
|
|
|
// MARK: - Event dispatch
|
|
|
|
func handleEvent(_ ev: VoiceCatEvent) {
|
|
switch ev.type {
|
|
case .channelList:
|
|
refreshChannels()
|
|
syncSelfChannel()
|
|
case .userJoined:
|
|
// ev.text = nickname, ev.channelId = the channel they joined (per voicecat.h).
|
|
if ev.userId != selfUserId && ev.channelId == currentChannelId {
|
|
EventFeedback.shared.play(.channelJoin)
|
|
EventFeedback.shared.speak("\(ev.text ?? "Someone") joined")
|
|
}
|
|
refreshUsers()
|
|
syncSelfChannel()
|
|
case .userLeft:
|
|
// Capture the leaving user's prior nickname/channel before refreshUsers() drops them.
|
|
if ev.userId != selfUserId,
|
|
let gone = users.first(where: { $0.id == ev.userId }),
|
|
gone.channelId == currentChannelId {
|
|
EventFeedback.shared.play(.channelLeave)
|
|
EventFeedback.shared.speak("\(gone.nickname) left")
|
|
}
|
|
refreshUsers()
|
|
syncSelfChannel()
|
|
case .userUpdated:
|
|
refreshUsers()
|
|
syncSelfChannel()
|
|
if let me = users.first(where: { $0.id == selfUserId }) {
|
|
applyServerMuteState(muted: me.serverMuted, deafened: me.serverDeafened)
|
|
}
|
|
case .textMessage:
|
|
let sender = users.first(where: { $0.id == ev.userId })?.nickname ?? "Unknown"
|
|
let body = ev.text ?? ""
|
|
let isSelf = ev.userId == selfUserId
|
|
let isPrivate = ev.textScope == .private
|
|
messages.append(ChatMessage(
|
|
timestamp: Date(timeIntervalSince1970: Double(ev.timestampUnixMs) / 1000),
|
|
senderName: sender,
|
|
text: body,
|
|
scope: ev.textScope))
|
|
EventFeedback.shared.play(isPrivate
|
|
? (isSelf ? .pmSent : .pmRecv)
|
|
: (isSelf ? .channelSent : .channelRecv))
|
|
if !isSelf {
|
|
EventFeedback.shared.speak(isPrivate
|
|
? "Private message from \(sender): \(body)"
|
|
: "\(sender): \(body)")
|
|
}
|
|
case .talkState:
|
|
let talking = ev.u32a != 0
|
|
if ev.userId == selfUserId {
|
|
EventFeedback.shared.play(talking ? .vaStart : .vaStop)
|
|
}
|
|
let who = users.first(where: { $0.id == ev.userId })?.nickname ?? "user \(ev.userId)"
|
|
addActivity(talking ? "\(who) started talking" : "\(who) stopped talking")
|
|
case .streamStarted:
|
|
// Our own SCREEN_AUDIO stream is live — begin draining the broadcast ring into it,
|
|
// in the stream's effective channel mode (downmix to mono if the channel is mono).
|
|
if ev.userId == selfUserId && ev.streamId == voiceState.screenStreamId {
|
|
let sid = voiceState.screenStreamId
|
|
let (r, cfg) = client.getStreamAudioConfig(userId: selfUserId, streamId: sid)
|
|
let channels: UInt32 = (r == .ok && cfg?.stereo == true) ? 2 : 1
|
|
let c = client
|
|
broadcastPump.beginFeeding(streamChannels: channels) { pcm, samples, ch in
|
|
c.feedPcm(streamId: sid, pcm: pcm, samplesPerChannel: samples, channels: ch)
|
|
}
|
|
addActivity("Sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
|
|
break
|
|
}
|
|
// A remote user started a stream — ensure the audio session is active so we can
|
|
// hear them even if we haven't joined voice ourselves.
|
|
if ev.userId != selfUserId {
|
|
do {
|
|
try AudioSessionManager.shared.ensureSessionActive()
|
|
} catch {
|
|
addActivity("Audio session activate failed: \(error)")
|
|
}
|
|
}
|
|
AudioSessionManager.shared.logSessionState("stream started (user \(ev.userId))")
|
|
addActivity("Stream started (user \(ev.userId))")
|
|
case .streamStopped:
|
|
addActivity("Stream stopped (user \(ev.userId))")
|
|
case .joinResult:
|
|
if ev.result == .ok {
|
|
currentChannelId = ev.channelId
|
|
addActivity("Joined channel")
|
|
refreshUsers()
|
|
} else {
|
|
addActivity("Join failed: \(ev.result.description)")
|
|
}
|
|
case .error:
|
|
addActivity("Error: \(ev.text ?? ev.result.description)")
|
|
case .genericResult:
|
|
if ev.result != .ok {
|
|
addActivity("Operation failed: \(ev.result.description)")
|
|
}
|
|
case .accountList:
|
|
accounts = client.listAccounts()
|
|
case .disconnected:
|
|
// Audible cue only — session teardown is driven elsewhere (AppState / UI).
|
|
EventFeedback.shared.play(ev.result == .ok ? .logout : .connectionLost)
|
|
EventFeedback.shared.speak(ev.result == .ok ? "Disconnected" : "Connection lost")
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
|
|
private func addActivity(_ text: String) {
|
|
activityLog.append(ActivityEntry(timestamp: Date(), text: text))
|
|
if activityLog.count > 500 { activityLog.removeFirst() }
|
|
}
|
|
|
|
// MARK: - Self-channel / server-mute sync
|
|
|
|
/// Sync currentChannelId from the self user's channelId in the user list. Mirrors macOS
|
|
/// MainWindowController.swift:461,491,522. The server auto-places every authed user into
|
|
/// the Lobby (channel 1) on connect, but without this sync currentChannelId stays 0 and
|
|
/// the mic button (gated on currentChannelId == 0) stays permanently dimmed.
|
|
private func syncSelfChannel() {
|
|
if let me = users.first(where: { $0.id == selfUserId }) {
|
|
currentChannelId = me.channelId
|
|
}
|
|
}
|
|
|
|
/// Apply server-side mute/deafen state — mirrors macOS MainWindowController.swift:693-700.
|
|
/// iOS was previously ignoring server mute/deafen entirely.
|
|
private func applyServerMuteState(muted: Bool, deafened: Bool) {
|
|
if muted && !voiceState.serverMuted { addActivity("You have been server-muted") }
|
|
if deafened && !voiceState.serverDeafened { addActivity("You have been server-deafened") }
|
|
if !muted && voiceState.serverMuted { addActivity("Server mute cleared") }
|
|
if !deafened && voiceState.serverDeafened { addActivity("Server deafen cleared") }
|
|
voiceState.serverMuted = muted
|
|
voiceState.serverDeafened = deafened
|
|
}
|
|
|
|
// MARK: - Data refresh
|
|
|
|
func refreshChannels() { channels = client.listChannels() }
|
|
func refreshUsers() { users = client.listUsers() }
|
|
func refreshDevices() { devices = client.listDevices(.input) }
|
|
|
|
// MARK: - Voice controls
|
|
|
|
func joinChannel(_ channelId: UInt32, password: String = "") {
|
|
client.joinChannel(channelId, password: password.isEmpty ? nil : password)
|
|
}
|
|
|
|
func leaveChannel() {
|
|
client.leaveChannel()
|
|
currentChannelId = 0
|
|
}
|
|
|
|
func startMicStream() {
|
|
AVAudioApplication.requestRecordPermission { [weak self] granted in
|
|
DispatchQueue.main.async {
|
|
guard let self else { return }
|
|
if granted {
|
|
self.doStartMicStream()
|
|
} else {
|
|
self.addActivity("Microphone permission denied — grant in Settings > Privacy > Microphone")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private func doStartMicStream() {
|
|
do {
|
|
try AudioSessionManager.shared.ensureSessionActive()
|
|
} catch {
|
|
addActivity("AVAudioSession activate failed: \(error)")
|
|
return
|
|
}
|
|
|
|
// Unified iOS path: the core is always external (set at connect via setExternalPlayback +
|
|
// every MIC stream external_feed), and `IOSAudioEngine` drives capture + playback. So the
|
|
// mic stream is just started with external_feed=true and the engine is told the mic is now
|
|
// active — no setExternalPlayback toggle, no audioRestart ordering, no VPIO/miniaudio fork.
|
|
let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic",
|
|
externalFeed: true)
|
|
let (result, streamId) = client.startStream(desc)
|
|
guard result == .ok else {
|
|
addActivity("Failed to start mic: \(result.description)")
|
|
return
|
|
}
|
|
voiceState.micActive = true
|
|
voiceState.localStreamId = streamId
|
|
EventFeedback.shared.play(.voiceOn)
|
|
|
|
let channels = IOSAudioRouter.shared.captureChannels.channelCount
|
|
if channels != 1 {
|
|
client.setCaptureChannels(streamId: streamId, channels: channels)
|
|
}
|
|
// Engage the mic: installs the input tap and (per preset) VPIO, in one engine rebuild.
|
|
IOSAudioEngine.shared.startMic(streamId: streamId, channels: channels)
|
|
}
|
|
|
|
func stopMicStream() {
|
|
// Disengage the mic (removes the tap + VPIO) but keep the engine running for any remaining
|
|
// remote audio. Then stop the core's MIC stream. The core stays external throughout — no
|
|
// setExternalPlayback toggle, no audioRestart.
|
|
IOSAudioEngine.shared.stopMic()
|
|
if voiceState.localStreamId != 0 {
|
|
client.stopStream(voiceState.localStreamId)
|
|
voiceState.localStreamId = 0
|
|
EventFeedback.shared.play(.voiceOff)
|
|
}
|
|
voiceState.micActive = false
|
|
voiceState.level = 0
|
|
// Do NOT deactivate the AVAudioSession here — the user may still want to hear
|
|
// remote audio (other people talking). The session is deactivated only when
|
|
// disconnecting from the server (see AppState.disconnect / .disconnected event).
|
|
}
|
|
|
|
// MARK: - Screen audio share
|
|
|
|
/// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream;
|
|
/// feeding begins on the resulting `.streamStarted` event (see handleEvent). The actual
|
|
/// system-audio capture happens in the ReplayKit upload extension (a separate process).
|
|
private func startScreenShare() {
|
|
guard voiceState.screenStreamId == 0 else { return }
|
|
guard currentChannelId != 0 else {
|
|
addActivity("Screen audio ignored — join a channel first")
|
|
return
|
|
}
|
|
let (result, streamId) = client.startStream(
|
|
StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Screen audio"))
|
|
if result == .ok {
|
|
voiceState.screenStreamId = streamId
|
|
voiceState.screenSharing = true
|
|
addActivity("Screen audio share starting…")
|
|
} else {
|
|
addActivity("Failed to start screen audio: \(result.description)")
|
|
}
|
|
}
|
|
|
|
/// Called when the broadcast ends (or on disconnect). Stops feeding and the stream.
|
|
private func stopScreenShare() {
|
|
broadcastPump.endFeeding()
|
|
if voiceState.screenStreamId != 0 {
|
|
client.stopStream(voiceState.screenStreamId)
|
|
voiceState.screenStreamId = 0
|
|
}
|
|
if voiceState.screenSharing {
|
|
voiceState.screenSharing = false
|
|
addActivity("Stopped sharing screen audio")
|
|
}
|
|
}
|
|
|
|
func setMute(_ muted: Bool, deafened: Bool) {
|
|
client.setSelfMute(micMuted: muted, deafened: deafened)
|
|
voiceState.selfMuted = muted
|
|
voiceState.selfDeafened = deafened
|
|
}
|
|
|
|
func setInputMode(_ mode: VoiceCatInputMode) {
|
|
client.setInputMode(mode)
|
|
voiceState.inputMode = mode
|
|
}
|
|
|
|
func setVadThreshold(_ threshold: Float) {
|
|
client.setVadThreshold(threshold)
|
|
voiceState.vadThreshold = threshold
|
|
}
|
|
|
|
private var pttEngaged = false
|
|
func setPushToTalk(_ active: Bool) {
|
|
client.setPushToTalk(active)
|
|
// Play the PTT cue only on the press transition (the gesture fires repeatedly while held).
|
|
if active && !pttEngaged { EventFeedback.shared.play(.ptt) }
|
|
pttEngaged = active
|
|
}
|
|
|
|
// MARK: - Text
|
|
|
|
func sendText(_ text: String, scope: VoiceCatTextScope, targetId: UInt32 = 0) {
|
|
client.sendText(scope: scope, targetId: targetId, text: text)
|
|
}
|
|
|
|
// MARK: - Admin
|
|
|
|
func kickUser(_ userId: UInt32, reason: String) {
|
|
client.kickUser(userId, reason: reason.isEmpty ? nil : reason)
|
|
}
|
|
|
|
func banUser(_ userId: UInt32, reason: String, expiresUnixMs: UInt64) {
|
|
client.banUser(userId, reason: reason.isEmpty ? nil : reason, expiresUnixMs: expiresUnixMs)
|
|
}
|
|
|
|
func moveUser(_ userId: UInt32, toChannel channelId: UInt32) {
|
|
client.moveUser(userId, toChannel: channelId)
|
|
}
|
|
|
|
func setPermissions(_ userId: UInt32, perms: Permissions) {
|
|
client.setPermission(userId, perms: perms)
|
|
}
|
|
|
|
func setServerMute(_ userId: UInt32, muted: Bool, deafened: Bool) {
|
|
client.setServerMute(userId, muted: muted, deafened: deafened)
|
|
}
|
|
|
|
func createChannel(_ info: ChannelEdit) {
|
|
client.createChannel(info)
|
|
}
|
|
|
|
func editChannel(_ info: ChannelEdit) {
|
|
client.editChannel(info)
|
|
}
|
|
|
|
func deleteChannel(_ channelId: UInt32) {
|
|
client.deleteChannel(channelId)
|
|
}
|
|
|
|
func fetchAccountList() {
|
|
client.requestAccountList()
|
|
}
|
|
|
|
func createAccount(username: String, password: String) {
|
|
client.createAccount(username, password: password)
|
|
}
|
|
|
|
func deleteAccount(username: String) {
|
|
client.deleteAccount(username)
|
|
}
|
|
|
|
func resetPassword(username: String, newPassword: String) {
|
|
client.resetPassword(username, newPassword: newPassword)
|
|
}
|
|
}
|