feat(macos): per-app audio selection for screen sharing

Let users choose what the SCREEN_AUDIO stream captures before sharing:
share everything, only selected apps, or all except selected apps, plus a
first-class "Exclude screen reader (VoiceOver) audio" toggle.

ScreenCaptureKit filters audio per application, so ScreenAudioCapture now
takes a ScreenAudioSelection and builds the matching SCContentFilter
(including:/excludingApplications:). New ScreenSharePickerSheet lists
running apps from SCShareableContent. iOS left untouched -- ReplayKit only
delivers the mixed system stream, so per-app filtering is impossible there.
This commit is contained in:
2026-06-21 13:35:01 +02:00
parent 6b7f06a282
commit c1e6f4f7ff
6 changed files with 389 additions and 16 deletions

View File

@@ -1,6 +1,25 @@
import AVFoundation
import ScreenCaptureKit
// Which apps' audio the SCREEN_AUDIO stream captures. ScreenCaptureKit filters audio at the
// *application* level (not per-window), so the selection is expressed as bundle IDs. The
// picker UI (ScreenSharePickerSheet) produces a `ScreenAudioSelection`; `start()` turns it
// into the matching `SCContentFilter`.
enum ScreenAudioScope: Equatable {
case entireDesktop // whole display the original behaviour
case onlyApps([String]) // capture only these bundle IDs
case allExcept([String]) // capture everything except these bundle IDs
}
struct ScreenAudioSelection: Equatable {
var scope: ScreenAudioScope = .entireDesktop
/// Drop the macOS screen-reader (VoiceOver) speech from the shared mix. Meaningful for
/// `.entireDesktop`/`.allExcept`; for `.onlyApps` the screen reader is already excluded.
var excludeScreenReader: Bool = false
static let `default` = ScreenAudioSelection()
}
// ScreenAudioCapture macOS system/desktop audio capture for the SCREEN_AUDIO stream.
//
// The macOS analog of the Windows WASAPI loopback path (docs/voice.md §9). ScreenCaptureKit
@@ -26,16 +45,27 @@ final class ScreenAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate {
private let onPcm: PcmHandler
private let channels: Int // 1 (mono) or 2 (stereo interleaved), matches the stream's mode
private let selection: ScreenAudioSelection
private let sampleQueue = DispatchQueue(label: "cat.voice.screenaudio.samples")
private var stream: SCStream?
/// Bundle IDs whose audio carries the macOS screen-reader speech. VoiceOver itself plus the
/// speech-synthesis daemon that actually renders the spoken audio the speech is usually
/// emitted by the daemon, not the VoiceOver app, so we exclude whichever are running.
static let screenReaderBundleIDs: Set<String> = [
"com.apple.VoiceOver",
"com.apple.VoiceOver4",
"com.apple.speech.speechsynthesisd",
]
/// Interleaved int16 carry-over between callbacks (ScreenCaptureKit buffers don't align to
/// 20 ms), drained in whole `frameSamplesPerChannel * channels` chunks. Only touched on
/// `sampleQueue`.
private var pending: [Int16] = []
init(channels: UInt32, onPcm: @escaping PcmHandler) {
init(channels: UInt32, selection: ScreenAudioSelection, onPcm: @escaping PcmHandler) {
self.channels = max(1, min(2, Int(channels)))
self.selection = selection
self.onPcm = onPcm
super.init()
}
@@ -46,7 +76,8 @@ final class ScreenAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate {
let content = try await SCShareableContent.current
guard let display = content.displays.first else { throw CaptureError.noDisplay }
let filter = SCContentFilter(display: display, excludingWindows: [])
let filter = Self.makeFilter(selection: selection, display: display,
apps: content.applications)
let config = SCStreamConfiguration()
config.capturesAudio = true
@@ -65,6 +96,39 @@ final class ScreenAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate {
self.stream = stream
}
/// Turn a `ScreenAudioSelection` into an `SCContentFilter` against the running apps.
/// ScreenCaptureKit filters audio per application, so we map bundle IDs SCRunningApplication.
private static func makeFilter(selection: ScreenAudioSelection, display: SCDisplay,
apps: [SCRunningApplication]) -> SCContentFilter {
func appsMatching(_ ids: Set<String>) -> [SCRunningApplication] {
apps.filter { ids.contains($0.bundleIdentifier) }
}
switch selection.scope {
case .onlyApps(let bundleIDs):
// Include-only already excludes everything else (the screen reader included), so the
// excludeScreenReader flag is moot in this mode.
return SCContentFilter(display: display,
including: appsMatching(Set(bundleIDs)),
exceptingWindows: [])
case .allExcept(let bundleIDs):
var ids = Set(bundleIDs)
if selection.excludeScreenReader { ids.formUnion(screenReaderBundleIDs) }
return SCContentFilter(display: display,
excludingApplications: appsMatching(ids),
exceptingWindows: [])
case .entireDesktop:
if selection.excludeScreenReader {
return SCContentFilter(display: display,
excludingApplications: appsMatching(screenReaderBundleIDs),
exceptingWindows: [])
}
return SCContentFilter(display: display, excludingWindows: [])
}
}
/// Stop capture and release the stream. Safe to call multiple times.
func stop() {
guard let stream else { return }

View File

@@ -0,0 +1,246 @@
import AppKit
import ScreenCaptureKit
// ScreenSharePickerSheet chooses *what* the SCREEN_AUDIO stream captures before sharing
// starts. ScreenCaptureKit filters audio per application (not per window), so the user picks
// a mode (everything / only-these / all-except-these) plus a set of apps, and a dedicated
// toggle to drop their own screen-reader (VoiceOver) speech from the mix.
//
// Mirrors the modal-sheet pattern used by the rest of the macOS client (UserPickerSheet,
// MoveUserSheet, ): an NSViewController presented via MainWindowController.presentSheet, with
// an `onComplete` callback that returns the chosen `ScreenAudioSelection` (or nil on cancel).
//
// The app list comes from `SCShareableContent.current`, fetched asynchronously that first
// access is also what surfaces the Screen Recording (TCC) prompt, which is why the picker is
// the natural place for it to appear, before any capture begins.
final class ScreenSharePickerSheet: NSViewController, NSTableViewDataSource, NSTableViewDelegate {
/// Called with the chosen selection, or `nil` if the user cancelled.
var onComplete: ((ScreenAudioSelection?) -> Void)?
private enum Mode: Int { case everything = 0, only = 1, except = 2 }
private struct AppEntry { let name: String; let bundleID: String; let icon: NSImage? }
private let initialSelection: ScreenAudioSelection
private var mode: Mode
private var excludeScreenReader: Bool
private var checked: Set<String> // bundle IDs ticked in the app table
private var apps: [AppEntry] = []
private let tableView = NSTableView()
private var modeControl: NSSegmentedControl?
private var screenReaderCheckbox: NSButton?
private var statusLabel: NSTextField?
init(selection: ScreenAudioSelection) {
self.initialSelection = selection
switch selection.scope {
case .entireDesktop: mode = .everything; checked = []
case .onlyApps(let ids): mode = .only; checked = Set(ids)
case .allExcept(let ids): mode = .except; checked = Set(ids)
}
self.excludeScreenReader = selection.excludeScreenReader
super.init(nibName: nil, bundle: nil)
}
required init?(coder: NSCoder) { fatalError() }
override func loadView() {
view = NSView(frame: NSRect(x: 0, y: 0, width: 360, height: 420))
}
override func viewDidLoad() {
super.viewDidLoad()
buildUI()
loadApps()
}
// MARK: - UI
private func buildUI() {
let titleLabel = NSTextField(labelWithString: "Choose what to share:")
titleLabel.font = .boldSystemFont(ofSize: 13)
titleLabel.setAccessibilityLabel("Choose what to share")
let modeControl = NSSegmentedControl(
labels: ["Everything", "Only selected", "All except selected"],
trackingMode: .selectOne, target: self, action: #selector(modeChanged))
modeControl.selectedSegment = mode.rawValue
modeControl.segmentDistribution = .fillEqually
modeControl.setAccessibilityLabel("Share mode")
self.modeControl = modeControl
let col = NSTableColumn(identifier: NSUserInterfaceItemIdentifier("app"))
tableView.addTableColumn(col)
tableView.headerView = nil
tableView.dataSource = self
tableView.delegate = self
tableView.rowHeight = 22
tableView.setAccessibilityLabel("Application list")
let scroll = NSScrollView()
scroll.documentView = tableView
scroll.hasVerticalScroller = true
scroll.borderType = .bezelBorder
scroll.translatesAutoresizingMaskIntoConstraints = false
scroll.setContentHuggingPriority(.defaultLow, for: .vertical)
let statusLabel = NSTextField(labelWithString: "Loading apps…")
statusLabel.textColor = .secondaryLabelColor
statusLabel.font = .systemFont(ofSize: 11)
self.statusLabel = statusLabel
let screenReaderCheckbox = NSButton(checkboxWithTitle: "Exclude screen reader (VoiceOver) audio",
target: self, action: #selector(screenReaderToggled))
screenReaderCheckbox.state = excludeScreenReader ? .on : .off
screenReaderCheckbox.toolTip = "Keep your VoiceOver speech out of the shared audio."
self.screenReaderCheckbox = screenReaderCheckbox
let cancelButton = NSButton(title: "Cancel", target: self, action: #selector(cancelClicked))
cancelButton.bezelStyle = .rounded
cancelButton.keyEquivalent = "\u{1b}" // Esc
let shareButton = NSButton(title: "Share", target: self, action: #selector(shareClicked))
shareButton.bezelStyle = .rounded
shareButton.keyEquivalent = "\r"
let buttonRow = NSStackView(views: [NSView(), cancelButton, shareButton])
buttonRow.orientation = .horizontal
buttonRow.spacing = 8
let stack = NSStackView(views: [titleLabel, modeControl, scroll, statusLabel,
screenReaderCheckbox, buttonRow])
stack.orientation = .vertical
stack.spacing = 8
stack.edgeInsets = NSEdgeInsets(top: 16, left: 16, bottom: 16, right: 16)
stack.translatesAutoresizingMaskIntoConstraints = false
view.addSubview(stack)
NSLayoutConstraint.activate([
stack.topAnchor.constraint(equalTo: view.topAnchor),
stack.leadingAnchor.constraint(equalTo: view.leadingAnchor),
stack.trailingAnchor.constraint(equalTo: view.trailingAnchor),
stack.bottomAnchor.constraint(equalTo: view.bottomAnchor),
])
updateEnabledStates()
}
/// Reflect the current mode: the app table only matters for only/except; the screen-reader
/// toggle is moot for `.only` (include-only already excludes the screen reader).
private func updateEnabledStates() {
let listActive = (mode != .everything)
tableView.isEnabled = listActive
tableView.alphaValue = listActive ? 1.0 : 0.45
screenReaderCheckbox?.isEnabled = (mode != .only)
}
private func loadApps() {
Task { @MainActor in
do {
let content = try await SCShareableContent.current
var seen = Set<String>()
var entries: [AppEntry] = []
for app in content.applications {
let bid = app.bundleIdentifier
guard !bid.isEmpty, !seen.contains(bid) else { continue }
// Hide our own app (its audio is already excluded) and the screen reader
// (handled by its own toggle).
if bid == Bundle.main.bundleIdentifier { continue }
if ScreenAudioCapture.screenReaderBundleIDs.contains(bid) { continue }
seen.insert(bid)
let name = app.applicationName.isEmpty ? bid : app.applicationName
let icon = NSRunningApplication
.runningApplications(withBundleIdentifier: bid).first?.icon
entries.append(AppEntry(name: name, bundleID: bid, icon: icon))
}
entries.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
self.apps = entries
self.statusLabel?.isHidden = true
self.tableView.reloadData()
} catch {
self.statusLabel?.stringValue = "Screen Recording permission needed to list apps — "
+ "grant it in System Settings ▸ Privacy & Security, then reopen this."
}
}
}
// MARK: - Actions
@objc private func modeChanged() {
mode = Mode(rawValue: modeControl?.selectedSegment ?? 0) ?? .everything
updateEnabledStates()
}
@objc private func screenReaderToggled() {
excludeScreenReader = (screenReaderCheckbox?.state == .on)
}
@objc private func appCheckboxToggled(_ sender: NSButton) {
let row = sender.tag
guard row >= 0, row < apps.count else { return }
let bid = apps[row].bundleID
if sender.state == .on { checked.insert(bid) } else { checked.remove(bid) }
}
@objc private func shareClicked() {
let scope: ScreenAudioScope
switch mode {
case .everything: scope = .entireDesktop
case .only: scope = .onlyApps(Array(checked))
case .except: scope = .allExcept(Array(checked))
}
dismiss(nil)
onComplete?(ScreenAudioSelection(scope: scope, excludeScreenReader: excludeScreenReader))
}
@objc private func cancelClicked() {
dismiss(nil)
onComplete?(nil)
}
// MARK: - NSTableViewDataSource / Delegate
func numberOfRows(in tableView: NSTableView) -> Int { apps.count }
func tableView(_ tableView: NSTableView, viewFor tableColumn: NSTableColumn?, row: Int) -> NSView? {
let app = apps[row]
let cell = AppCheckboxCell()
cell.checkbox.title = app.name
cell.checkbox.state = checked.contains(app.bundleID) ? .on : .off
cell.checkbox.isEnabled = (mode != .everything)
cell.checkbox.tag = row
cell.checkbox.target = self
cell.checkbox.action = #selector(appCheckboxToggled(_:))
cell.iconView.image = app.icon
return cell
}
// Selecting a row shouldn't visually highlight interaction is via the checkbox.
func tableView(_ tableView: NSTableView, shouldSelectRow row: Int) -> Bool { false }
}
// One row: a leading app icon and a checkbox titled with the app name.
private final class AppCheckboxCell: NSTableCellView {
let iconView = NSImageView()
let checkbox = NSButton(checkboxWithTitle: "", target: nil, action: nil)
init() {
super.init(frame: .zero)
iconView.translatesAutoresizingMaskIntoConstraints = false
checkbox.translatesAutoresizingMaskIntoConstraints = false
addSubview(iconView)
addSubview(checkbox)
NSLayoutConstraint.activate([
iconView.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 4),
iconView.centerYAnchor.constraint(equalTo: centerYAnchor),
iconView.widthAnchor.constraint(equalToConstant: 16),
iconView.heightAnchor.constraint(equalToConstant: 16),
checkbox.leadingAnchor.constraint(equalTo: iconView.trailingAnchor, constant: 6),
checkbox.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -4),
checkbox.centerYAnchor.constraint(equalTo: centerYAnchor),
])
}
required init?(coder: NSCoder) { fatalError() }
}

View File

@@ -31,6 +31,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
internal var micStreamId: UInt32 = 0
private var screenStreamId: UInt32 = 0
private var screenCapture: ScreenAudioCapture?
// Last app/exclusion choice from the share picker; reused as the default next time.
private var screenAudioSelection: ScreenAudioSelection = .default
internal var pttKeyCode: UInt16 = 0x60 // F8
private var pttMonitor: Any?
private var serverMuted = false
@@ -720,17 +722,15 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
@objc private func screenAudioClicked() {
if screenStreamId == 0 {
// Announce the stream now; ScreenCaptureKit capture starts once the server's
// StreamAnnounceResult lands (the .streamStarted event), when the effective audio
// config and thus the channel count to capture is known. See startScreenCapture.
let (result, streamId) = client.startStream(StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Desktop audio"))
if result == .ok {
screenStreamId = streamId
setShareScreenButton(active: true)
addActivity("Starting screen audio share…")
} else {
addActivity("Failed to start screen audio: \(result)")
// Let the user pick what to share (apps to include/exclude, screen-reader audio)
// before we announce anything. The picker remembers the previous choice.
let sheet = ScreenSharePickerSheet(selection: screenAudioSelection)
sheet.onComplete = { [weak self] selection in
guard let self, let selection else { return } // nil = cancelled
self.screenAudioSelection = selection
self.beginScreenAudioShare()
}
presentSheet(sheet)
} else {
stopScreenCapture()
client.stopStream(screenStreamId)
@@ -740,6 +740,21 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
}
/// Announce the SCREEN_AUDIO stream with the chosen selection in hand. ScreenCaptureKit
/// capture starts once the server's StreamAnnounceResult lands (the .streamStarted event),
/// when the effective audio config and thus the channel count is known. See
/// startScreenCapture.
private func beginScreenAudioShare() {
let (result, streamId) = client.startStream(StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Desktop audio"))
if result == .ok {
screenStreamId = streamId
setShareScreenButton(active: true)
addActivity("Starting screen audio share…")
} else {
addActivity("Failed to start screen audio: \(result)")
}
}
private func setShareScreenButton(active: Bool) {
shareScreenButton?.title = active ? "Stop Screen Audio" : "Share Screen Audio"
shareScreenButton?.image = NSImage(
@@ -760,7 +775,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
let channels: UInt32 = (cfgResult == .ok && cfg?.stereo == true) ? 2 : 1
let streamId = screenStreamId
let capture = ScreenAudioCapture(channels: channels) { [weak self] pcm, samples, ch in
let capture = ScreenAudioCapture(channels: channels, selection: screenAudioSelection) { [weak self] pcm, samples, ch in
self?.client.feedPcm(streamId: streamId, pcm: pcm, samplesPerChannel: samples, channels: ch)
}
screenCapture = capture
@@ -768,7 +783,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
Task { @MainActor in
do {
try await capture.start()
addActivity("Started sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
addActivity("Started sharing screen audio (\(channels == 2 ? "stereo" : "mono"))"
+ "\(Self.scopeSuffix(for: screenAudioSelection))")
} catch {
// Most commonly: Screen Recording permission denied. Roll back the stream.
screenCapture = nil
@@ -788,6 +804,21 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
screenCapture = nil
}
/// A short human-readable description of the share scope for the activity log.
private static func scopeSuffix(for selection: ScreenAudioSelection) -> String {
var parts: [String] = []
switch selection.scope {
case .entireDesktop: break
case .onlyApps(let ids): if !ids.isEmpty { parts.append("only \(ids.count) app(s)") }
case .allExcept(let ids): if !ids.isEmpty { parts.append("excluding \(ids.count) app(s)") }
}
// For .onlyApps the screen reader is already excluded, so don't claim it twice.
if selection.excludeScreenReader {
if case .onlyApps = selection.scope {} else { parts.append("no screen reader") }
}
return parts.isEmpty ? "" : "" + parts.joined(separator: ", ")
}
@objc private func muteChanged() {
let muted = muteButton?.state == .on
let deafened = deafenButton?.state == .on