feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit
Implement system/desktop audio sharing on the Apple clients, feeding the existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No C++/protocol/codec changes -- the core was already ready (the Windows-only loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits for fed PCM). Audio only; video is dropped. macOS (in-process): - ScreenAudioCapture.swift drives an audio-only SCStream (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's mono/stereo mode, and calls feedPcm. Capture starts on the self .streamStarted event (effective config known then). Wired into MainWindowController.screenAudioClicked(). iOS (forward-to-host, single session): - VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does not link libvoicecat. - Host BroadcastAudioPump drains the ring (reacting to the extension's Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing to mono when the channel is mono. Screen audio appears as a second stream of the same user; no credentials persisted. UI is RPSystemBroadcastPicker View in VoiceControlsView. Removes the speculative BroadcastCredentials. Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
@@ -29,6 +29,8 @@ struct VoiceState {
|
||||
var level: Float = 0.0
|
||||
var currentDeviceId: String?
|
||||
var localStreamId: UInt32 = 0
|
||||
var screenSharing = false
|
||||
var screenStreamId: UInt32 = 0
|
||||
}
|
||||
|
||||
// MARK: - SessionState
|
||||
@@ -49,6 +51,10 @@ final class SessionState {
|
||||
var accounts: [Account] = []
|
||||
var devices: [Device] = []
|
||||
|
||||
/// Host side of iOS screen-audio sharing — drains the broadcast extension's App Group ring
|
||||
/// and feeds the SCREEN_AUDIO stream this session owns. See BroadcastAudioPump.
|
||||
private let broadcastPump = BroadcastAudioPump()
|
||||
|
||||
init(client: VoiceCatClient, selfUserId: UInt32, permissions: Permissions) {
|
||||
self.client = client
|
||||
self.selfUserId = selfUserId
|
||||
@@ -64,9 +70,13 @@ final class SessionState {
|
||||
client.onLevel = { [weak self] _, rms in
|
||||
Task { @MainActor [weak self] in self?.voiceState.level = rms }
|
||||
}
|
||||
broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() }
|
||||
broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() }
|
||||
broadcastPump.start()
|
||||
}
|
||||
|
||||
deinit {
|
||||
broadcastPump.stop()
|
||||
MainActor.assumeIsolated {
|
||||
AudioSessionManager.shared.client = nil
|
||||
}
|
||||
@@ -100,6 +110,19 @@ final class SessionState {
|
||||
let who = users.first(where: { $0.id == ev.userId })?.nickname ?? "user \(ev.userId)"
|
||||
addActivity(talking ? "\(who) started talking" : "\(who) stopped talking")
|
||||
case .streamStarted:
|
||||
// Our own SCREEN_AUDIO stream is live — begin draining the broadcast ring into it,
|
||||
// in the stream's effective channel mode (downmix to mono if the channel is mono).
|
||||
if ev.userId == selfUserId && ev.streamId == voiceState.screenStreamId {
|
||||
let sid = voiceState.screenStreamId
|
||||
let (r, cfg) = client.getStreamAudioConfig(userId: selfUserId, streamId: sid)
|
||||
let channels: UInt32 = (r == .ok && cfg?.stereo == true) ? 2 : 1
|
||||
let c = client
|
||||
broadcastPump.beginFeeding(streamChannels: channels) { pcm, samples, ch in
|
||||
c.feedPcm(streamId: sid, pcm: pcm, samplesPerChannel: samples, channels: ch)
|
||||
}
|
||||
addActivity("Sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
|
||||
break
|
||||
}
|
||||
// A remote user started a stream — ensure the audio session is active so we can
|
||||
// hear them even if we haven't joined voice ourselves.
|
||||
if ev.userId != selfUserId {
|
||||
@@ -236,6 +259,41 @@ final class SessionState {
|
||||
// disconnecting from the server (see AppState.disconnect / .disconnected event).
|
||||
}
|
||||
|
||||
// MARK: - Screen audio share
|
||||
|
||||
/// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream;
|
||||
/// feeding begins on the resulting `.streamStarted` event (see handleEvent). The actual
|
||||
/// system-audio capture happens in the ReplayKit upload extension (a separate process).
|
||||
private func startScreenShare() {
|
||||
guard voiceState.screenStreamId == 0 else { return }
|
||||
guard currentChannelId != 0 else {
|
||||
addActivity("Screen audio ignored — join a channel first")
|
||||
return
|
||||
}
|
||||
let (result, streamId) = client.startStream(
|
||||
StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Screen audio"))
|
||||
if result == .ok {
|
||||
voiceState.screenStreamId = streamId
|
||||
voiceState.screenSharing = true
|
||||
addActivity("Screen audio share starting…")
|
||||
} else {
|
||||
addActivity("Failed to start screen audio: \(result.description)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Called when the broadcast ends (or on disconnect). Stops feeding and the stream.
|
||||
private func stopScreenShare() {
|
||||
broadcastPump.endFeeding()
|
||||
if voiceState.screenStreamId != 0 {
|
||||
client.stopStream(voiceState.screenStreamId)
|
||||
voiceState.screenStreamId = 0
|
||||
}
|
||||
if voiceState.screenSharing {
|
||||
voiceState.screenSharing = false
|
||||
addActivity("Stopped sharing screen audio")
|
||||
}
|
||||
}
|
||||
|
||||
func setMute(_ muted: Bool, deafened: Bool) {
|
||||
client.setSelfMute(micMuted: muted, deafened: deafened)
|
||||
voiceState.selfMuted = muted
|
||||
|
||||
Reference in New Issue
Block a user