feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit
Implement system/desktop audio sharing on the Apple clients, feeding the existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No C++/protocol/codec changes -- the core was already ready (the Windows-only loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits for fed PCM). Audio only; video is dropped. macOS (in-process): - ScreenAudioCapture.swift drives an audio-only SCStream (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's mono/stereo mode, and calls feedPcm. Capture starts on the self .streamStarted event (effective config known then). Wired into MainWindowController.screenAudioClicked(). iOS (forward-to-host, single session): - VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does not link libvoicecat. - Host BroadcastAudioPump drains the ring (reacting to the extension's Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing to mono when the channel is mono. Screen audio appears as a second stream of the same user; no credentials persisted. UI is RPSystemBroadcastPicker View in VoiceControlsView. Removes the speculative BroadcastCredentials. Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
@@ -30,6 +30,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
canMoveUsers: false, canAdminAccounts: false, isAdmin: false)
|
||||
internal var micStreamId: UInt32 = 0
|
||||
private var screenStreamId: UInt32 = 0
|
||||
private var screenCapture: ScreenAudioCapture?
|
||||
internal var pttKeyCode: UInt16 = 0x60 // F8
|
||||
private var pttMonitor: Any?
|
||||
private var serverMuted = false
|
||||
@@ -477,6 +478,10 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
}
|
||||
|
||||
case .streamStarted:
|
||||
// Our own SCREEN_AUDIO stream is now live on the server — begin capture.
|
||||
if event.userId == selfUserId && event.streamId == screenStreamId {
|
||||
startScreenCapture()
|
||||
}
|
||||
guard let u = users[event.userId], u.channelId == currentChannelId else { break }
|
||||
let streams = client.listUserStreams(event.userId)
|
||||
let kind = streams.first(where: { $0.id == event.streamId })?.kind
|
||||
@@ -601,6 +606,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
channelTree = []; channelOutlineView.reloadData()
|
||||
displayedUsers = []; userTableView.reloadData()
|
||||
users.removeAll(); talkingUsers.removeAll()
|
||||
stopScreenCapture()
|
||||
currentChannelId = 0; micStreamId = 0; screenStreamId = 0
|
||||
composeField.isEnabled = false; sendButton.isEnabled = false
|
||||
joinVoiceButton?.isEnabled = false
|
||||
@@ -708,26 +714,70 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
|
||||
@objc private func screenAudioClicked() {
|
||||
if screenStreamId == 0 {
|
||||
// Announce the stream now; ScreenCaptureKit capture starts once the server's
|
||||
// StreamAnnounceResult lands (the .streamStarted event), when the effective audio
|
||||
// config — and thus the channel count to capture — is known. See startScreenCapture.
|
||||
let (result, streamId) = client.startStream(StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Desktop audio"))
|
||||
if result == .ok {
|
||||
screenStreamId = streamId
|
||||
shareScreenButton?.title = "Stop Screen Audio"
|
||||
shareScreenButton?.image = NSImage(systemSymbolName: "rectangle.on.rectangle.angled.fill",
|
||||
accessibilityDescription: "Stop Screen Audio")
|
||||
addActivity("Started sharing screen audio")
|
||||
setShareScreenButton(active: true)
|
||||
addActivity("Starting screen audio share…")
|
||||
} else {
|
||||
addActivity("Failed to start screen audio: \(result)")
|
||||
}
|
||||
} else {
|
||||
stopScreenCapture()
|
||||
client.stopStream(screenStreamId)
|
||||
screenStreamId = 0
|
||||
shareScreenButton?.title = "Share Screen Audio"
|
||||
shareScreenButton?.image = NSImage(systemSymbolName: "rectangle.on.rectangle.angled",
|
||||
accessibilityDescription: "Share Screen Audio")
|
||||
setShareScreenButton(active: false)
|
||||
addActivity("Stopped sharing screen audio")
|
||||
}
|
||||
}
|
||||
|
||||
private func setShareScreenButton(active: Bool) {
|
||||
shareScreenButton?.title = active ? "Stop Screen Audio" : "Share Screen Audio"
|
||||
shareScreenButton?.image = NSImage(
|
||||
systemSymbolName: active ? "rectangle.on.rectangle.angled.fill" : "rectangle.on.rectangle.angled",
|
||||
accessibilityDescription: active ? "Stop Screen Audio" : "Share Screen Audio")
|
||||
}
|
||||
|
||||
/// Start ScreenCaptureKit capture for our own SCREEN_AUDIO stream. Called from the
|
||||
/// `.streamStarted` event handler, where the effective audio config is available. Captures
|
||||
/// in the channel's mode (stereo when the channel is stereo) and feeds 20 ms PCM frames.
|
||||
private func startScreenCapture() {
|
||||
guard screenStreamId != 0, screenCapture == nil else { return }
|
||||
let (cfgResult, cfg) = client.getStreamAudioConfig(userId: selfUserId, streamId: screenStreamId)
|
||||
let channels: UInt32 = (cfgResult == .ok && cfg?.stereo == true) ? 2 : 1
|
||||
|
||||
let streamId = screenStreamId
|
||||
let capture = ScreenAudioCapture(channels: channels) { [weak self] pcm, samples, ch in
|
||||
self?.client.feedPcm(streamId: streamId, pcm: pcm, samplesPerChannel: samples, channels: ch)
|
||||
}
|
||||
screenCapture = capture
|
||||
|
||||
Task { @MainActor in
|
||||
do {
|
||||
try await capture.start()
|
||||
addActivity("Started sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
|
||||
} catch {
|
||||
// Most commonly: Screen Recording permission denied. Roll back the stream.
|
||||
screenCapture = nil
|
||||
if screenStreamId != 0 {
|
||||
client.stopStream(screenStreamId)
|
||||
screenStreamId = 0
|
||||
setShareScreenButton(active: false)
|
||||
}
|
||||
addActivity("Screen audio capture failed — grant Screen Recording in System "
|
||||
+ "Settings ▸ Privacy & Security, then try again. (\(error))")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func stopScreenCapture() {
|
||||
screenCapture?.stop()
|
||||
screenCapture = nil
|
||||
}
|
||||
|
||||
@objc private func muteChanged() {
|
||||
let muted = muteButton?.state == .on
|
||||
let deafened = deafenButton?.state == .on
|
||||
@@ -999,6 +1049,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
}
|
||||
}
|
||||
client.setPushToTalk(false)
|
||||
stopScreenCapture()
|
||||
if screenStreamId != 0 { client.stopStream(screenStreamId) }
|
||||
if micStreamId != 0 { client.stopStream(micStreamId) }
|
||||
client.onLevel = nil
|
||||
|
||||
Reference in New Issue
Block a user