Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit

Browse Source 
Implement system/desktop audio sharing on the Apple clients, feeding the
existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No
C++/protocol/codec changes -- the core was already ready (the Windows-only
loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits
for fed PCM). Audio only; video is dropped.

macOS (in-process):
- ScreenAudioCapture.swift drives an audio-only SCStream
  (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's
  mono/stereo mode, and calls feedPcm. Capture starts on the self
  .streamStarted event (effective config known then). Wired into
  MainWindowController.screenAudioClicked().

iOS (forward-to-host, single session):
- VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes
  .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and
  writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does
  not link libvoicecat.
- Host BroadcastAudioPump drains the ring (reacting to the extension's
  Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing
  to mono when the channel is mono. Screen audio appears as a second stream
  of the same user; no credentials persisted. UI is RPSystemBroadcastPicker
  View in VoiceControlsView. Removes the speculative BroadcastCredentials.

Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
Talon
2026-06-21 00:14:31 +02:00
parent 6ab78fa792
commit 8c90e250f0
17 changed files with 985 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
clients/apple/macOS/VoiceCatMac.xcodeproj/project.pbxproj
View File

@@ -30,6 +30,7 @@
AAAA00000000000000000046 /* PrivateMessageWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA00000000000000000045 /* PrivateMessageWindowController.swift */; };
AAAA00000000000000000048 /* UserPickerSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA00000000000000000047 /* UserPickerSheet.swift */; };
AAAA0000000000000000004A /* SettingsWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA00000000000000000049 /* SettingsWindowController.swift */; };
AAAA0000000000000000004C /* ScreenAudioCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA0000000000000000004B /* ScreenAudioCapture.swift */; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
@@ -42,6 +43,7 @@
AAAA00000000000000000018 /* ServerListStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerListStore.swift; sourceTree = "<group>"; };
AAAA00000000000000000019 /* ConnectWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectWindowController.swift; sourceTree = "<group>"; };
AAAA0000000000000000001A /* MainWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainWindowController.swift; sourceTree = "<group>"; };
AAAA0000000000000000004B /* ScreenAudioCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScreenAudioCapture.swift; sourceTree = "<group>"; };
AAAA0000000000000000001B /* AddServerSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddServerSheet.swift; sourceTree = "<group>"; };
AAAA0000000000000000001C /* ServerIdentitySheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerIdentitySheet.swift; sourceTree = "<group>"; };
AAAA0000000000000000001D /* PasswordPromptSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PasswordPromptSheet.swift; sourceTree = "<group>"; };
@@ -89,12 +91,21 @@
AAAA00000000000000000015 /* main.swift */,
AAAA00000000000000000016 /* AppDelegate.swift */,
AAAA00000000000000000004 /* Models */,
AAAA0000000000000000004D /* Audio */,
AAAA00000000000000000005 /* Windows */,
AAAA00000000000000000006 /* Sheets */,
);
path = VoiceCatMac;
sourceTree = "<group>";
};
AAAA0000000000000000004D /* Audio */ = {
isa = PBXGroup;
children = (
AAAA0000000000000000004B /* ScreenAudioCapture.swift */,
);
path = Audio;
sourceTree = "<group>";
};
AAAA00000000000000000004 /* Models */ = {
isa = PBXGroup;
children = (
@@ -231,6 +242,7 @@
AAAA00000000000000000046 /* PrivateMessageWindowController.swift in Sources */,
AAAA00000000000000000048 /* UserPickerSheet.swift in Sources */,
AAAA0000000000000000004A /* SettingsWindowController.swift in Sources */,
AAAA0000000000000000004C /* ScreenAudioCapture.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};

158
clients/apple/macOS/VoiceCatMac/Audio/ScreenAudioCapture.swift Normal file
View File

@@ -0,0 +1,158 @@
import AVFoundation
import ScreenCaptureKit
// ScreenAudioCapture — macOS system/desktop audio capture for the SCREEN_AUDIO stream.
//
// The macOS analog of the Windows WASAPI loopback path (docs/voice.md §9). ScreenCaptureKit
// (macOS 13+) captures whatever the system is playing; we convert each audio CMSampleBuffer
// (Float32) → int16 interleaved and push 20 ms frames (960 samples/channel @ 48 kHz) into the
// core via `vc_stream_feed_pcm` (exposed as `VoiceCatClient.feedPcm`). The core then runs the
// same Opus-encode → media-AEAD → UDP path as any other stream — only the *source* is
// platform-specific (architecture.md §4).
//
// Audio-only: we request a 2×2 video plane at 1 fps purely because SCStream needs a video
// configuration, and we never add a `.screen` output — only `.audio`. `excludesCurrentProcess
// Audio` prevents the self-echo loop of re-capturing our own incoming voice mix.
//
// `feedPcm` is thread-safe (any thread), so we forward straight from the sample-handler queue.
final class ScreenAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate {
/// Receives a full 20 ms frame: (interleaved int16 PCM, samplesPerChannel = 960, channels).
typealias PcmHandler = (UnsafePointer<Int16>, Int, UInt32) -> Void
enum CaptureError: Error { case noDisplay }
private static let frameSamplesPerChannel = 960 // 20 ms @ 48 kHz
private let onPcm: PcmHandler
private let channels: Int // 1 (mono) or 2 (stereo interleaved), matches the stream's mode
private let sampleQueue = DispatchQueue(label: "cat.voice.screenaudio.samples")
private var stream: SCStream?
/// Interleaved int16 carry-over between callbacks (ScreenCaptureKit buffers don't align to
/// 20 ms), drained in whole `frameSamplesPerChannel * channels` chunks. Only touched on
/// `sampleQueue`.
private var pending: [Int16] = []
init(channels: UInt32, onPcm: @escaping PcmHandler) {
self.channels = max(1, min(2, Int(channels)))
self.onPcm = onPcm
super.init()
}
/// Begin capture. Throws if Screen Recording permission is denied (the first
/// `SCShareableContent.current` access is what surfaces the TCC prompt) or no display exists.
func start() async throws {
let content = try await SCShareableContent.current
guard let display = content.displays.first else { throw CaptureError.noDisplay }
let filter = SCContentFilter(display: display, excludingWindows: [])
let config = SCStreamConfiguration()
config.capturesAudio = true
config.excludesCurrentProcessAudio = true
config.sampleRate = 48000
config.channelCount = channels
// SCStream requires a video config even when we only consume audio — keep it minimal.
config.width = 2
config.height = 2
config.minimumFrameInterval = CMTime(value: 1, timescale: 1) // ~1 fps
config.queueDepth = 6
let stream = SCStream(filter: filter, configuration: config, delegate: self)
try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
try await stream.startCapture()
self.stream = stream
}
/// Stop capture and release the stream. Safe to call multiple times.
func stop() {
guard let stream else { return }
self.stream = nil
Task { try? await stream.stopCapture() }
}
// MARK: - SCStreamOutput
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
of type: SCStreamOutputType) {
guard type == .audio, CMSampleBufferDataIsReady(sampleBuffer) else { return }
guard let fmt = sampleBuffer.formatDescription,
let asbd = fmt.audioStreamBasicDescription else { return }
// ScreenCaptureKit always delivers Float32 PCM; bail on anything unexpected.
guard asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0 else { return }
let nonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0
let srcChannels = max(1, Int(asbd.mChannelsPerFrame))
try? sampleBuffer.withAudioBufferList { ablPtr, _ in
convert(ablPtr, nonInterleaved: nonInterleaved, srcChannels: srcChannels)
}
}
// MARK: - Conversion (called on sampleQueue)
private func convert(_ abl: UnsafeMutableAudioBufferListPointer,
nonInterleaved: Bool, srcChannels: Int) {
guard let first = abl.first, first.mData != nil else { return }
let out = channels
var interleaved: [Int16]
if nonInterleaved {
// One buffer per source channel; each is `frames` Float32 samples.
let frames = Int(first.mDataByteSize) / MemoryLayout<Float>.size
if frames == 0 { return }
let ch0 = first.mData!.assumingMemoryBound(to: Float.self)
let ch1: UnsafePointer<Float>? = (abl.count > 1)
? UnsafePointer(abl[1].mData!.assumingMemoryBound(to: Float.self)) : nil
interleaved = [Int16](repeating: 0, count: frames * out)
for i in 0..<frames {
let l = ch0[i]
let r = ch1?[i] ?? l
if out == 2 {
interleaved[i * 2] = Self.f2i(l)
interleaved[i * 2 + 1] = Self.f2i(srcChannels >= 2 ? r : l)
} else {
interleaved[i] = Self.f2i(srcChannels >= 2 ? (l + r) * 0.5 : l)
}
}
} else {
// Single interleaved Float32 buffer, srcChannels wide.
let total = Int(first.mDataByteSize) / MemoryLayout<Float>.size
let frames = total / srcChannels
if frames == 0 { return }
let src = first.mData!.assumingMemoryBound(to: Float.self)
interleaved = [Int16](repeating: 0, count: frames * out)
for i in 0..<frames {
let l = src[i * srcChannels]
let r = srcChannels >= 2 ? src[i * srcChannels + 1] : l
if out == 2 {
interleaved[i * 2] = Self.f2i(l)
interleaved[i * 2 + 1] = Self.f2i(r)
} else {
interleaved[i] = Self.f2i(srcChannels >= 2 ? (l + r) * 0.5 : l)
}
}
}
emit(interleaved)
}
/// Accumulate interleaved int16 and fire `onPcm` for every whole 20 ms frame.
private func emit(_ interleaved: [Int16]) {
pending.append(contentsOf: interleaved)
let full = Self.frameSamplesPerChannel * channels
while pending.count >= full {
pending.withUnsafeBufferPointer { buf in
onPcm(buf.baseAddress!, Self.frameSamplesPerChannel, UInt32(channels))
}
pending.removeFirst(full)
}
}
private static func f2i(_ f: Float) -> Int16 {
let v = max(-1.0, min(1.0, f)) * 32767.0
return Int16(v.rounded())
}
}

65
clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift
View File

@@ -30,6 +30,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
canMoveUsers: false, canAdminAccounts: false, isAdmin: false)
internal var micStreamId: UInt32 = 0
private var screenStreamId: UInt32 = 0
private var screenCapture: ScreenAudioCapture?
internal var pttKeyCode: UInt16 = 0x60 // F8
private var pttMonitor: Any?
private var serverMuted = false
@@ -477,6 +478,10 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
case .streamStarted:
// Our own SCREEN_AUDIO stream is now live on the server — begin capture.
if event.userId == selfUserId && event.streamId == screenStreamId {
startScreenCapture()
}
guard let u = users[event.userId], u.channelId == currentChannelId else { break }
let streams = client.listUserStreams(event.userId)
let kind = streams.first(where: { $0.id == event.streamId })?.kind
@@ -601,6 +606,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
channelTree = []; channelOutlineView.reloadData()
displayedUsers = []; userTableView.reloadData()
users.removeAll(); talkingUsers.removeAll()
stopScreenCapture()
currentChannelId = 0; micStreamId = 0; screenStreamId = 0
composeField.isEnabled = false; sendButton.isEnabled = false
joinVoiceButton?.isEnabled = false
@@ -708,26 +714,70 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
@objc private func screenAudioClicked() {
if screenStreamId == 0 {
// Announce the stream now; ScreenCaptureKit capture starts once the server's
// StreamAnnounceResult lands (the .streamStarted event), when the effective audio
// config — and thus the channel count to capture — is known. See startScreenCapture.
let (result, streamId) = client.startStream(StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Desktop audio"))
if result == .ok {
screenStreamId = streamId
shareScreenButton?.title = "Stop Screen Audio"
shareScreenButton?.image = NSImage(systemSymbolName: "rectangle.on.rectangle.angled.fill",
accessibilityDescription: "Stop Screen Audio")
addActivity("Started sharing screen audio")
setShareScreenButton(active: true)
addActivity("Starting screen audio share…")
} else {
addActivity("Failed to start screen audio: \(result)")
}
} else {
stopScreenCapture()
client.stopStream(screenStreamId)
screenStreamId = 0
shareScreenButton?.title = "Share Screen Audio"
shareScreenButton?.image = NSImage(systemSymbolName: "rectangle.on.rectangle.angled",
accessibilityDescription: "Share Screen Audio")
setShareScreenButton(active: false)
addActivity("Stopped sharing screen audio")
}
}
private func setShareScreenButton(active: Bool) {
shareScreenButton?.title = active ? "Stop Screen Audio" : "Share Screen Audio"
shareScreenButton?.image = NSImage(
systemSymbolName: active ? "rectangle.on.rectangle.angled.fill" : "rectangle.on.rectangle.angled",
accessibilityDescription: active ? "Stop Screen Audio" : "Share Screen Audio")
}
/// Start ScreenCaptureKit capture for our own SCREEN_AUDIO stream. Called from the
/// `.streamStarted` event handler, where the effective audio config is available. Captures
/// in the channel's mode (stereo when the channel is stereo) and feeds 20 ms PCM frames.
private func startScreenCapture() {
guard screenStreamId != 0, screenCapture == nil else { return }
let (cfgResult, cfg) = client.getStreamAudioConfig(userId: selfUserId, streamId: screenStreamId)
let channels: UInt32 = (cfgResult == .ok && cfg?.stereo == true) ? 2 : 1
let streamId = screenStreamId
let capture = ScreenAudioCapture(channels: channels) { [weak self] pcm, samples, ch in
self?.client.feedPcm(streamId: streamId, pcm: pcm, samplesPerChannel: samples, channels: ch)
}
screenCapture = capture
Task { @MainActor in
do {
try await capture.start()
addActivity("Started sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
} catch {
// Most commonly: Screen Recording permission denied. Roll back the stream.
screenCapture = nil
if screenStreamId != 0 {
client.stopStream(screenStreamId)
screenStreamId = 0
setShareScreenButton(active: false)
}
addActivity("Screen audio capture failed — grant Screen Recording in System "
+ "Settings ▸ Privacy & Security, then try again. (\(error))")
}
}
}
private func stopScreenCapture() {
screenCapture?.stop()
screenCapture = nil
}
@objc private func muteChanged() {
let muted = muteButton?.state == .on
let deafened = deafenButton?.state == .on
@@ -999,6 +1049,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
}
client.setPushToTalk(false)
stopScreenCapture()
if screenStreamId != 0 { client.stopStream(screenStreamId) }
if micStreamId != 0 { client.stopStream(micStreamId) }
client.onLevel = nil