feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit
Implement system/desktop audio sharing on the Apple clients, feeding the existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No C++/protocol/codec changes -- the core was already ready (the Windows-only loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits for fed PCM). Audio only; video is dropped. macOS (in-process): - ScreenAudioCapture.swift drives an audio-only SCStream (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's mono/stereo mode, and calls feedPcm. Capture starts on the self .streamStarted event (effective config known then). Wired into MainWindowController.screenAudioClicked(). iOS (forward-to-host, single session): - VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does not link libvoicecat. - Host BroadcastAudioPump drains the ring (reacting to the extension's Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing to mono when the channel is mono. Screen audio appears as a second stream of the same user; no credentials persisted. UI is RPSystemBroadcastPicker View in VoiceControlsView. Removes the speculative BroadcastCredentials. Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
178
clients/apple/iOS/Shared/BroadcastAudioRing.swift
Normal file
178
clients/apple/iOS/Shared/BroadcastAudioRing.swift
Normal file
@@ -0,0 +1,178 @@
|
||||
import Foundation
|
||||
import Darwin
|
||||
|
||||
// Darwin notification names the extension posts and the host observes, so the host can react to
|
||||
// broadcast start/stop promptly instead of only polling the ring's active flag. Shared (compiled
|
||||
// into both targets) so the names can't drift.
|
||||
enum BroadcastNotification {
|
||||
static let started = "cat.voice.VoiceCat.broadcast.started"
|
||||
static let finished = "cat.voice.VoiceCat.broadcast.finished"
|
||||
}
|
||||
|
||||
// BroadcastAudioRing — cross-process single-producer/single-consumer int16 PCM ring over an
|
||||
// mmap'd file in the shared App Group container. Compiled into BOTH the host app and the
|
||||
// ReplayKit broadcast upload extension (docs/voice.md §9, iOS detail).
|
||||
//
|
||||
// producer = the broadcast extension's RPBroadcastSampleHandler (captured system audio)
|
||||
// consumer = the host app's BroadcastAudioPump (drains and feeds the core via feedPcm)
|
||||
//
|
||||
// Why a hand-rolled ring instead of Swift's `Atomic`: the storage lives in shared memory mapped
|
||||
// into two processes, so the synchronization words must sit in that mapping — Swift's managed
|
||||
// atomics can't. For strict SPSC, aligned 64-bit monotonic indices with full memory barriers
|
||||
// (`OSMemoryBarrier`) give correct acquire/release ordering. The extension does NOT link
|
||||
// libvoicecat — it only writes PCM here; all encode/crypto/UDP happens in the host.
|
||||
//
|
||||
// Lifecycle: the host opens the ring at connect (and thus initializes the header first); the
|
||||
// extension opens it later when a broadcast starts. On a rising edge of `isActive` the host
|
||||
// calls `drainStale()` to discard any pre-roll, then drains in whole 20 ms frames.
|
||||
final class BroadcastAudioRing {
|
||||
|
||||
static let appGroupId = "group.cat.voice.VoiceCat"
|
||||
|
||||
enum RingError: Error { case noContainer, openFailed, mapFailed }
|
||||
|
||||
// Header layout (byte offsets into the mmap). Indices are 8-byte aligned; mmap is
|
||||
// page-aligned so offsets 24/32 satisfy that.
|
||||
private static let magic: UInt32 = 0x5643_4252 // "VCBR"
|
||||
private static let version: UInt32 = 1
|
||||
private static let headerBytes = 64
|
||||
/// 1 second of 48 kHz stereo int16 — ample slack for ~10 ms host drains.
|
||||
static let capacitySamples = 48_000 * 2
|
||||
|
||||
private static let offMagic = 0
|
||||
private static let offVersion = 4
|
||||
private static let offChannels = 8
|
||||
private static let offSampleRate = 12
|
||||
private static let offActive = 16
|
||||
private static let offWrite = 24
|
||||
private static let offRead = 32
|
||||
|
||||
private let fd: Int32
|
||||
private let mapBase: UnsafeMutableRawPointer
|
||||
private let mapSize: Int
|
||||
private let data: UnsafeMutablePointer<Int16>
|
||||
private let capacity: Int
|
||||
|
||||
init() throws {
|
||||
guard let container = FileManager.default.containerURL(
|
||||
forSecurityApplicationGroupIdentifier: Self.appGroupId) else {
|
||||
throw RingError.noContainer
|
||||
}
|
||||
let dir = container.appendingPathComponent("voicecat", isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
||||
let path = dir.appendingPathComponent("broadcast_audio.ring").path
|
||||
|
||||
capacity = Self.capacitySamples
|
||||
mapSize = Self.headerBytes + capacity * MemoryLayout<Int16>.size
|
||||
|
||||
let f = open(path, O_RDWR | O_CREAT, 0o644)
|
||||
guard f >= 0 else { throw RingError.openFailed }
|
||||
if ftruncate(f, off_t(mapSize)) != 0 { close(f); throw RingError.openFailed }
|
||||
|
||||
let p = mmap(nil, mapSize, PROT_READ | PROT_WRITE, MAP_SHARED, f, 0)
|
||||
guard let p, p != MAP_FAILED else { close(f); throw RingError.mapFailed }
|
||||
|
||||
fd = f
|
||||
mapBase = p
|
||||
data = (p + Self.headerBytes).assumingMemoryBound(to: Int16.self)
|
||||
|
||||
// First opener initializes the header (host opens first, before any broadcast).
|
||||
if load32(Self.offMagic) != Self.magic {
|
||||
store32(Self.offWrite, 0); store32(Self.offWrite + 4, 0)
|
||||
store32(Self.offRead, 0); store32(Self.offRead + 4, 0)
|
||||
store32(Self.offChannels, 0)
|
||||
store32(Self.offSampleRate, 0)
|
||||
store32(Self.offActive, 0)
|
||||
store32(Self.offVersion, Self.version)
|
||||
OSMemoryBarrier()
|
||||
store32(Self.offMagic, Self.magic)
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
munmap(mapBase, mapSize)
|
||||
close(fd)
|
||||
}
|
||||
|
||||
// MARK: - Header accessors
|
||||
|
||||
var isActive: Bool { OSMemoryBarrier(); return load32(Self.offActive) != 0 }
|
||||
var channels: UInt32 { load32(Self.offChannels) }
|
||||
var sampleRate: UInt32 { load32(Self.offSampleRate) }
|
||||
|
||||
/// Producer: advertise the canonical capture format and toggle the active flag. Called from
|
||||
/// `broadcastStarted`/`broadcastFinished`.
|
||||
func setActive(_ active: Bool, channels: UInt32 = 0, sampleRate: UInt32 = 0) {
|
||||
if active {
|
||||
store32(Self.offChannels, channels)
|
||||
store32(Self.offSampleRate, sampleRate)
|
||||
}
|
||||
OSMemoryBarrier()
|
||||
store32(Self.offActive, active ? 1 : 0)
|
||||
}
|
||||
|
||||
// MARK: - Producer (extension)
|
||||
|
||||
/// Append interleaved int16 samples. Drops the whole chunk if it doesn't fit — skipping a
|
||||
/// chunk is better than tearing a frame. Single producer only.
|
||||
func push(_ samples: UnsafeBufferPointer<Int16>) {
|
||||
let n = samples.count
|
||||
guard n > 0, n <= capacity, let src = samples.baseAddress else { return }
|
||||
let w = load64(Self.offWrite) // producer owns the write index
|
||||
let r = loadAcquire64(Self.offRead)
|
||||
if capacity - Int(w &- r) < n { return } // full: drop
|
||||
var idx = Int(w % UInt64(capacity))
|
||||
var off = 0
|
||||
var rem = n
|
||||
while rem > 0 {
|
||||
let chunk = min(rem, capacity - idx)
|
||||
(data + idx).update(from: src + off, count: chunk)
|
||||
idx = (idx + chunk) % capacity
|
||||
off += chunk
|
||||
rem -= chunk
|
||||
}
|
||||
storeRelease64(Self.offWrite, w &+ UInt64(n))
|
||||
}
|
||||
|
||||
// MARK: - Consumer (host)
|
||||
|
||||
/// Read up to `out.count` interleaved int16 samples. Returns the number read. Single
|
||||
/// consumer only.
|
||||
func read(into out: UnsafeMutableBufferPointer<Int16>) -> Int {
|
||||
guard let dst = out.baseAddress else { return 0 }
|
||||
let r = load64(Self.offRead) // consumer owns the read index
|
||||
let w = loadAcquire64(Self.offWrite)
|
||||
let available = Int(w &- r)
|
||||
if available <= 0 { return 0 }
|
||||
let n = min(available, out.count)
|
||||
var idx = Int(r % UInt64(capacity))
|
||||
var off = 0
|
||||
var rem = n
|
||||
while rem > 0 {
|
||||
let chunk = min(rem, capacity - idx)
|
||||
(dst + off).update(from: data + idx, count: chunk)
|
||||
idx = (idx + chunk) % capacity
|
||||
off += chunk
|
||||
rem -= chunk
|
||||
}
|
||||
storeRelease64(Self.offRead, r &+ UInt64(n))
|
||||
return n
|
||||
}
|
||||
|
||||
/// Consumer: discard everything currently buffered (catch the read index up to write).
|
||||
func drainStale() { storeRelease64(Self.offRead, loadAcquire64(Self.offWrite)) }
|
||||
|
||||
// MARK: - Memory-ordered accessors
|
||||
|
||||
private func ptr32(_ off: Int) -> UnsafeMutablePointer<UInt32> {
|
||||
(mapBase + off).assumingMemoryBound(to: UInt32.self)
|
||||
}
|
||||
private func ptr64(_ off: Int) -> UnsafeMutablePointer<UInt64> {
|
||||
(mapBase + off).assumingMemoryBound(to: UInt64.self)
|
||||
}
|
||||
private func load32(_ off: Int) -> UInt32 { ptr32(off).pointee }
|
||||
private func store32(_ off: Int, _ v: UInt32) { ptr32(off).pointee = v }
|
||||
private func load64(_ off: Int) -> UInt64 { ptr64(off).pointee }
|
||||
private func loadAcquire64(_ off: Int) -> UInt64 { let v = ptr64(off).pointee; OSMemoryBarrier(); return v }
|
||||
private func storeRelease64(_ off: Int, _ v: UInt64) { OSMemoryBarrier(); ptr64(off).pointee = v }
|
||||
}
|
||||
Reference in New Issue
Block a user