Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit

Browse Source 
Implement system/desktop audio sharing on the Apple clients, feeding the
existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No
C++/protocol/codec changes -- the core was already ready (the Windows-only
loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits
for fed PCM). Audio only; video is dropped.

macOS (in-process):
- ScreenAudioCapture.swift drives an audio-only SCStream
  (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's
  mono/stereo mode, and calls feedPcm. Capture starts on the self
  .streamStarted event (effective config known then). Wired into
  MainWindowController.screenAudioClicked().

iOS (forward-to-host, single session):
- VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes
  .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and
  writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does
  not link libvoicecat.
- Host BroadcastAudioPump drains the ring (reacting to the extension's
  Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing
  to mono when the channel is mono. Screen audio appears as a second stream
  of the same user; no credentials persisted. UI is RPSystemBroadcastPicker
  View in VoiceControlsView. Removes the speculative BroadcastCredentials.

Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
Talon
2026-06-21 00:14:31 +02:00
parent 6ab78fa792
commit 8c90e250f0
17 changed files with 985 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
clients/apple/iOS/VoiceCatiOS/AppState.swift
View File

@@ -153,7 +153,6 @@ final class AppState {
guard let client = connectingClient else { break }
let perms = client.getPermissions()
let newSession = SessionState(client: client, selfUserId: ev.userId, permissions: perms)
ServerListStore.shared.writeBroadcastCredentials(server: server, nickname: nil)
connectingClient = nil
isConnecting = false
connectStatus = ""

159
clients/apple/iOS/VoiceCatiOS/BroadcastAudioPump.swift Normal file
View File

@@ -0,0 +1,159 @@
import Foundation
// BroadcastAudioPump — host side of iOS screen-audio sharing (docs/voice.md §9, iOS detail).
//
// Drains the App Group shared-memory ring written by the broadcast upload extension and hands
// whole 20 ms frames to a feed closure (which calls VoiceCatClient.feedPcm). The host app owns
// the SCREEN_AUDIO stream, so screen audio appears as a second stream of the SAME user — exactly
// like the Windows/macOS desktop-audio share, and a single session (no separate connection).
//
// It reacts to the extension's Darwin notifications for prompt start/stop, and the drain timer
// also watches the ring's active flag as a safety net if a notification is missed. The ring
// always carries stereo; we downmix to mono when the stream's effective config is mono.
//
// Not @MainActor: the drain runs on a background queue (feedPcm is thread-safe). The start/stop
// callbacks are dispatched to main so they can drive @MainActor SessionState.
final class BroadcastAudioPump {
/// The host should start the SCREEN_AUDIO stream (a broadcast became active).
var onBroadcastStarted: (() -> Void)?
/// The host should stop the SCREEN_AUDIO stream (the broadcast ended).
var onBroadcastFinished: (() -> Void)?
private static let frameSamplesPerChannel = 960 // 20 ms @ 48 kHz
private let queue = DispatchQueue(label: "cat.voice.broadcast.pump")
private var ring: BroadcastAudioRing?
private var timer: DispatchSourceTimer?
private var feed: ((UnsafePointer<Int16>, Int, UInt32) -> Void)?
private var streamChannels = 1
private var ringChannels = 2
private var pending: [Int16] = [] // interleaved at ringChannels width
private var scratch = [Int16](repeating: 0, count: 8192)
private var started = false
// MARK: - Lifecycle (host connect/disconnect)
func start() {
guard !started else { return }
ring = try? BroadcastAudioRing()
started = true
registerDarwin()
// Host reconnected while a broadcast is still running — pick it up.
if ring?.isActive == true { onBroadcastStarted?() }
}
func stop() {
guard started else { return }
started = false
unregisterDarwin()
endFeeding()
ring = nil
}
// MARK: - Feeding (driven by the host once the stream is live)
/// Begin draining the ring into `feed`. Called after the SCREEN_AUDIO stream's
/// `.streamStarted` event, when its effective channel count is known.
func beginFeeding(streamChannels: UInt32,
feed: @escaping (UnsafePointer<Int16>, Int, UInt32) -> Void) {
queue.async {
self.streamChannels = max(1, min(2, Int(streamChannels)))
self.ringChannels = max(1, Int(self.ring?.channels ?? 2))
self.feed = feed
self.pending.removeAll(keepingCapacity: true)
self.ring?.drainStale() // discard pre-roll buffered before we were ready
self.startTimer()
}
}
func endFeeding() {
queue.async {
self.timer?.cancel()
self.timer = nil
self.feed = nil
self.pending.removeAll(keepingCapacity: true)
}
}
// MARK: - Drain
private func startTimer() {
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now(), repeating: .milliseconds(10), leeway: .milliseconds(2))
t.setEventHandler { [weak self] in self?.drain() }
timer = t
t.resume()
}
private func drain() {
guard let ring, let feed else { return }
// Safety net: broadcast ended but we missed the Darwin note.
if !ring.isActive {
DispatchQueue.main.async { [weak self] in self?.onBroadcastFinished?() }
return
}
while true {
let got = scratch.withUnsafeMutableBufferPointer { ring.read(into: $0) }
if got == 0 { break }
pending.append(contentsOf: scratch[0..<got])
if got < scratch.count { break }
}
emitFrames(feed)
}
private func emitFrames(_ feed: (UnsafePointer<Int16>, Int, UInt32) -> Void) {
let n = Self.frameSamplesPerChannel
let rc = ringChannels
let sc = streamChannels
let inFrame = n * rc
while pending.count >= inFrame {
if sc == rc {
pending.withUnsafeBufferPointer { feed($0.baseAddress!, n, UInt32(sc)) }
} else if sc == 1 && rc == 2 {
var mono = [Int16](repeating: 0, count: n)
pending.withUnsafeBufferPointer { buf in
let p = buf.baseAddress!
for i in 0..<n { mono[i] = Int16((Int(p[i * 2]) + Int(p[i * 2 + 1])) / 2) }
}
mono.withUnsafeBufferPointer { feed($0.baseAddress!, n, 1) }
} else if sc == 2 && rc == 1 {
var stereo = [Int16](repeating: 0, count: n * 2)
pending.withUnsafeBufferPointer { buf in
let p = buf.baseAddress!
for i in 0..<n { stereo[i * 2] = p[i]; stereo[i * 2 + 1] = p[i] }
}
stereo.withUnsafeBufferPointer { feed($0.baseAddress!, n, 2) }
}
pending.removeFirst(inFrame)
}
}
// MARK: - Darwin notifications
private func registerDarwin() {
let observer = Unmanaged.passUnretained(self).toOpaque()
let center = CFNotificationCenterGetDarwinNotifyCenter()
let callback: CFNotificationCallback = { _, observer, name, _, _ in
guard let observer, let name else { return }
let pump = Unmanaged<BroadcastAudioPump>.fromOpaque(observer).takeUnretainedValue()
let raw = name.rawValue as String
DispatchQueue.main.async {
if raw == BroadcastNotification.started { pump.onBroadcastStarted?() }
else if raw == BroadcastNotification.finished { pump.onBroadcastFinished?() }
}
}
CFNotificationCenterAddObserver(center, observer, callback,
BroadcastNotification.started as CFString, nil, .deliverImmediately)
CFNotificationCenterAddObserver(center, observer, callback,
BroadcastNotification.finished as CFString, nil, .deliverImmediately)
}
private func unregisterDarwin() {
CFNotificationCenterRemoveEveryObserver(
CFNotificationCenterGetDarwinNotifyCenter(),
Unmanaged.passUnretained(self).toOpaque())
}
deinit { if started { unregisterDarwin() } }
}

21
clients/apple/iOS/VoiceCatiOS/BroadcastCredentials.swift
View File

@@ -1,21 +0,0 @@
import Foundation
struct BroadcastCredentials: Codable {
var host: String
var port: Int
var authMode: String
var username: String
var tofuPinsPath: String
static func loadFromAppGroup() -> BroadcastCredentials? {
let groupId = "group.cat.voice.VoiceCat"
guard let container = FileManager.default.containerURL(
forSecurityApplicationGroupIdentifier: groupId)
else { return nil }
let url = container
.appendingPathComponent("voicecat", isDirectory: true)
.appendingPathComponent("broadcast_credentials.json")
guard let data = try? Data(contentsOf: url) else { return nil }
return try? JSONDecoder().decode(BroadcastCredentials.self, from: data)
}
}

14
clients/apple/iOS/VoiceCatiOS/ServerListStore.swift
View File

@@ -89,18 +89,4 @@ final class ServerListStore {
SecItemDelete(query as CFDictionary)
}
// MARK: - Broadcast credentials (shared with ReplayKit extension)
func writeBroadcastCredentials(server: SavedServer, nickname: String?) {
let creds = BroadcastCredentials(
host: server.host,
port: Int(server.port),
authMode: server.authMode.rawValue,
username: server.authMode == .password ? server.savedUsername : (nickname ?? ""),
tofuPinsPath: tofuStorePath
)
guard let data = try? JSONEncoder().encode(creds) else { return }
let url = voicecatDir.appendingPathComponent("broadcast_credentials.json")
try? data.write(to: url, options: .atomic)
}
}

58
clients/apple/iOS/VoiceCatiOS/SessionState.swift
View File

@@ -29,6 +29,8 @@ struct VoiceState {
var level: Float = 0.0
var currentDeviceId: String?
var localStreamId: UInt32 = 0
var screenSharing = false
var screenStreamId: UInt32 = 0
}
// MARK: - SessionState
@@ -49,6 +51,10 @@ final class SessionState {
var accounts: [Account] = []
var devices: [Device] = []
/// Host side of iOS screen-audio sharing — drains the broadcast extension's App Group ring
/// and feeds the SCREEN_AUDIO stream this session owns. See BroadcastAudioPump.
private let broadcastPump = BroadcastAudioPump()
init(client: VoiceCatClient, selfUserId: UInt32, permissions: Permissions) {
self.client = client
self.selfUserId = selfUserId
@@ -64,9 +70,13 @@ final class SessionState {
client.onLevel = { [weak self] _, rms in
Task { @MainActor [weak self] in self?.voiceState.level = rms }
}
broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() }
broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() }
broadcastPump.start()
}
deinit {
broadcastPump.stop()
MainActor.assumeIsolated {
AudioSessionManager.shared.client = nil
}
@@ -100,6 +110,19 @@ final class SessionState {
let who = users.first(where: { $0.id == ev.userId })?.nickname ?? "user \(ev.userId)"
addActivity(talking ? "\(who) started talking" : "\(who) stopped talking")
case .streamStarted:
// Our own SCREEN_AUDIO stream is live — begin draining the broadcast ring into it,
// in the stream's effective channel mode (downmix to mono if the channel is mono).
if ev.userId == selfUserId && ev.streamId == voiceState.screenStreamId {
let sid = voiceState.screenStreamId
let (r, cfg) = client.getStreamAudioConfig(userId: selfUserId, streamId: sid)
let channels: UInt32 = (r == .ok && cfg?.stereo == true) ? 2 : 1
let c = client
broadcastPump.beginFeeding(streamChannels: channels) { pcm, samples, ch in
c.feedPcm(streamId: sid, pcm: pcm, samplesPerChannel: samples, channels: ch)
}
addActivity("Sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
break
}
// A remote user started a stream — ensure the audio session is active so we can
// hear them even if we haven't joined voice ourselves.
if ev.userId != selfUserId {
@@ -236,6 +259,41 @@ final class SessionState {
// disconnecting from the server (see AppState.disconnect / .disconnected event).
}
// MARK: - Screen audio share
/// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream;
/// feeding begins on the resulting `.streamStarted` event (see handleEvent). The actual
/// system-audio capture happens in the ReplayKit upload extension (a separate process).
private func startScreenShare() {
guard voiceState.screenStreamId == 0 else { return }
guard currentChannelId != 0 else {
addActivity("Screen audio ignored — join a channel first")
return
}
let (result, streamId) = client.startStream(
StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Screen audio"))
if result == .ok {
voiceState.screenStreamId = streamId
voiceState.screenSharing = true
addActivity("Screen audio share starting…")
} else {
addActivity("Failed to start screen audio: \(result.description)")
}
}
/// Called when the broadcast ends (or on disconnect). Stops feeding and the stream.
private func stopScreenShare() {
broadcastPump.endFeeding()
if voiceState.screenStreamId != 0 {
client.stopStream(voiceState.screenStreamId)
voiceState.screenStreamId = 0
}
if voiceState.screenSharing {
voiceState.screenSharing = false
addActivity("Stopped sharing screen audio")
}
}
func setMute(_ muted: Bool, deafened: Bool) {
client.setSelfMute(micMuted: muted, deafened: deafened)
voiceState.selfMuted = muted

29
clients/apple/iOS/VoiceCatiOS/Views/VoiceControlsView.swift
View File

@@ -1,5 +1,6 @@
import SwiftUI
import VoiceCatCore
import ReplayKit
struct VoiceControlsView: View {
@Bindable var session: SessionState
@@ -58,6 +59,14 @@ struct VoiceControlsView: View {
.disabled(!session.voiceState.micActive)
.accessibilityLabel(session.voiceState.selfDeafened ? "Undeafen" : "Deafen")
// Share screen audio (ReplayKit system broadcast picker). The picker launches the
// broadcast upload extension; the host's BroadcastAudioPump then owns + feeds the
// SCREEN_AUDIO stream. Tinted while sharing.
BroadcastPickerButton(isSharing: session.voiceState.screenSharing)
.frame(width: 32, height: 32)
.accessibilityLabel(session.voiceState.screenSharing
? "Stop sharing screen audio" : "Share screen audio")
// Disconnect
Button(role: .destructive) {
session.stopMicStream()
@@ -108,6 +117,26 @@ private struct PTTButton: View {
}
}
// MARK: - Broadcast picker
/// Wraps `RPSystemBroadcastPickerView` (which contains its own button) and points it at our
/// broadcast upload extension. Tapping it shows the system broadcast picker; the user starts the
/// broadcast and our extension launches.
private struct BroadcastPickerButton: UIViewRepresentable {
let isSharing: Bool
func makeUIView(context: Context) -> RPSystemBroadcastPickerView {
let picker = RPSystemBroadcastPickerView(frame: CGRect(x: 0, y: 0, width: 32, height: 32))
picker.preferredExtension = "cat.voice.VoiceCatiOS.broadcast"
picker.showsMicrophoneButton = false
return picker
}
func updateUIView(_ uiView: RPSystemBroadcastPickerView, context: Context) {
uiView.tintColor = isSharing ? .systemGreen : .label
}
}
// MARK: - Level Meter
private struct LevelMeterView: View {