feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit

Implement system/desktop audio sharing on the Apple clients, feeding the
existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No
C++/protocol/codec changes -- the core was already ready (the Windows-only
loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits
for fed PCM). Audio only; video is dropped.

macOS (in-process):
- ScreenAudioCapture.swift drives an audio-only SCStream
  (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's
  mono/stereo mode, and calls feedPcm. Capture starts on the self
  .streamStarted event (effective config known then). Wired into
  MainWindowController.screenAudioClicked().

iOS (forward-to-host, single session):
- VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes
  .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and
  writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does
  not link libvoicecat.
- Host BroadcastAudioPump drains the ring (reacting to the extension's
  Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing
  to mono when the channel is mono. Screen audio appears as a second stream
  of the same user; no credentials persisted. UI is RPSystemBroadcastPicker
  View in VoiceControlsView. Removes the speculative BroadcastCredentials.

Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
2026-06-21 00:14:31 +02:00
parent 6ab78fa792
commit 8c90e250f0
17 changed files with 985 additions and 69 deletions

View File

@@ -0,0 +1,178 @@
import Foundation
import Darwin
// Darwin notification names the extension posts and the host observes, so the host can react to
// broadcast start/stop promptly instead of only polling the ring's active flag. Shared (compiled
// into both targets) so the names can't drift.
enum BroadcastNotification {
static let started = "cat.voice.VoiceCat.broadcast.started"
static let finished = "cat.voice.VoiceCat.broadcast.finished"
}
// BroadcastAudioRing cross-process single-producer/single-consumer int16 PCM ring over an
// mmap'd file in the shared App Group container. Compiled into BOTH the host app and the
// ReplayKit broadcast upload extension (docs/voice.md §9, iOS detail).
//
// producer = the broadcast extension's RPBroadcastSampleHandler (captured system audio)
// consumer = the host app's BroadcastAudioPump (drains and feeds the core via feedPcm)
//
// Why a hand-rolled ring instead of Swift's `Atomic`: the storage lives in shared memory mapped
// into two processes, so the synchronization words must sit in that mapping Swift's managed
// atomics can't. For strict SPSC, aligned 64-bit monotonic indices with full memory barriers
// (`OSMemoryBarrier`) give correct acquire/release ordering. The extension does NOT link
// libvoicecat it only writes PCM here; all encode/crypto/UDP happens in the host.
//
// Lifecycle: the host opens the ring at connect (and thus initializes the header first); the
// extension opens it later when a broadcast starts. On a rising edge of `isActive` the host
// calls `drainStale()` to discard any pre-roll, then drains in whole 20 ms frames.
final class BroadcastAudioRing {
static let appGroupId = "group.cat.voice.VoiceCat"
enum RingError: Error { case noContainer, openFailed, mapFailed }
// Header layout (byte offsets into the mmap). Indices are 8-byte aligned; mmap is
// page-aligned so offsets 24/32 satisfy that.
private static let magic: UInt32 = 0x5643_4252 // "VCBR"
private static let version: UInt32 = 1
private static let headerBytes = 64
/// 1 second of 48 kHz stereo int16 ample slack for ~10 ms host drains.
static let capacitySamples = 48_000 * 2
private static let offMagic = 0
private static let offVersion = 4
private static let offChannels = 8
private static let offSampleRate = 12
private static let offActive = 16
private static let offWrite = 24
private static let offRead = 32
private let fd: Int32
private let mapBase: UnsafeMutableRawPointer
private let mapSize: Int
private let data: UnsafeMutablePointer<Int16>
private let capacity: Int
init() throws {
guard let container = FileManager.default.containerURL(
forSecurityApplicationGroupIdentifier: Self.appGroupId) else {
throw RingError.noContainer
}
let dir = container.appendingPathComponent("voicecat", isDirectory: true)
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
let path = dir.appendingPathComponent("broadcast_audio.ring").path
capacity = Self.capacitySamples
mapSize = Self.headerBytes + capacity * MemoryLayout<Int16>.size
let f = open(path, O_RDWR | O_CREAT, 0o644)
guard f >= 0 else { throw RingError.openFailed }
if ftruncate(f, off_t(mapSize)) != 0 { close(f); throw RingError.openFailed }
let p = mmap(nil, mapSize, PROT_READ | PROT_WRITE, MAP_SHARED, f, 0)
guard let p, p != MAP_FAILED else { close(f); throw RingError.mapFailed }
fd = f
mapBase = p
data = (p + Self.headerBytes).assumingMemoryBound(to: Int16.self)
// First opener initializes the header (host opens first, before any broadcast).
if load32(Self.offMagic) != Self.magic {
store32(Self.offWrite, 0); store32(Self.offWrite + 4, 0)
store32(Self.offRead, 0); store32(Self.offRead + 4, 0)
store32(Self.offChannels, 0)
store32(Self.offSampleRate, 0)
store32(Self.offActive, 0)
store32(Self.offVersion, Self.version)
OSMemoryBarrier()
store32(Self.offMagic, Self.magic)
}
}
deinit {
munmap(mapBase, mapSize)
close(fd)
}
// MARK: - Header accessors
var isActive: Bool { OSMemoryBarrier(); return load32(Self.offActive) != 0 }
var channels: UInt32 { load32(Self.offChannels) }
var sampleRate: UInt32 { load32(Self.offSampleRate) }
/// Producer: advertise the canonical capture format and toggle the active flag. Called from
/// `broadcastStarted`/`broadcastFinished`.
func setActive(_ active: Bool, channels: UInt32 = 0, sampleRate: UInt32 = 0) {
if active {
store32(Self.offChannels, channels)
store32(Self.offSampleRate, sampleRate)
}
OSMemoryBarrier()
store32(Self.offActive, active ? 1 : 0)
}
// MARK: - Producer (extension)
/// Append interleaved int16 samples. Drops the whole chunk if it doesn't fit skipping a
/// chunk is better than tearing a frame. Single producer only.
func push(_ samples: UnsafeBufferPointer<Int16>) {
let n = samples.count
guard n > 0, n <= capacity, let src = samples.baseAddress else { return }
let w = load64(Self.offWrite) // producer owns the write index
let r = loadAcquire64(Self.offRead)
if capacity - Int(w &- r) < n { return } // full: drop
var idx = Int(w % UInt64(capacity))
var off = 0
var rem = n
while rem > 0 {
let chunk = min(rem, capacity - idx)
(data + idx).update(from: src + off, count: chunk)
idx = (idx + chunk) % capacity
off += chunk
rem -= chunk
}
storeRelease64(Self.offWrite, w &+ UInt64(n))
}
// MARK: - Consumer (host)
/// Read up to `out.count` interleaved int16 samples. Returns the number read. Single
/// consumer only.
func read(into out: UnsafeMutableBufferPointer<Int16>) -> Int {
guard let dst = out.baseAddress else { return 0 }
let r = load64(Self.offRead) // consumer owns the read index
let w = loadAcquire64(Self.offWrite)
let available = Int(w &- r)
if available <= 0 { return 0 }
let n = min(available, out.count)
var idx = Int(r % UInt64(capacity))
var off = 0
var rem = n
while rem > 0 {
let chunk = min(rem, capacity - idx)
(dst + off).update(from: data + idx, count: chunk)
idx = (idx + chunk) % capacity
off += chunk
rem -= chunk
}
storeRelease64(Self.offRead, r &+ UInt64(n))
return n
}
/// Consumer: discard everything currently buffered (catch the read index up to write).
func drainStale() { storeRelease64(Self.offRead, loadAcquire64(Self.offWrite)) }
// MARK: - Memory-ordered accessors
private func ptr32(_ off: Int) -> UnsafeMutablePointer<UInt32> {
(mapBase + off).assumingMemoryBound(to: UInt32.self)
}
private func ptr64(_ off: Int) -> UnsafeMutablePointer<UInt64> {
(mapBase + off).assumingMemoryBound(to: UInt64.self)
}
private func load32(_ off: Int) -> UInt32 { ptr32(off).pointee }
private func store32(_ off: Int, _ v: UInt32) { ptr32(off).pointee = v }
private func load64(_ off: Int) -> UInt64 { ptr64(off).pointee }
private func loadAcquire64(_ off: Int) -> UInt64 { let v = ptr64(off).pointee; OSMemoryBarrier(); return v }
private func storeRelease64(_ off: Int, _ v: UInt64) { OSMemoryBarrier(); ptr64(off).pointee = v }
}

View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDisplayName</key>
<string>VoiceCat Screen Audio</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>0.0.1</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>NSExtension</key>
<dict>
<key>NSExtensionPointIdentifier</key>
<string>com.apple.broadcast-services-upload</string>
<key>NSExtensionPrincipalClass</key>
<string>$(PRODUCT_MODULE_NAME).SampleHandler</string>
<key>RPBroadcastProcessMode</key>
<string>BroadcastUpload</string>
</dict>
</dict>
</plist>

View File

@@ -0,0 +1,88 @@
import ReplayKit
import AVFoundation
// SampleHandler the ReplayKit broadcast upload extension entry point (docs/voice.md §9, iOS).
//
// This runs in a SEPARATE process with a ~50 MB memory cap. It captures system/app audio
// (`.audioApp`), drops video and mic buffers, converts each chunk to the core's canonical
// format (48 kHz, int16, stereo interleaved) with AVAudioConverter, and writes it into the
// App Group shared-memory ring. The host app's BroadcastAudioPump drains the ring and feeds the
// already-connected VoiceCatClient so all Opus/AEAD/UDP work happens in the host, and the
// extension stays tiny and well inside the memory budget (no libvoicecat here).
class SampleHandler: RPBroadcastSampleHandler {
private var ring: BroadcastAudioRing?
private var converter: AVAudioConverter?
private var inputFormat: AVAudioFormat?
private let outputFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 48_000, channels: 2, interleaved: true)!
override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) {
ring = try? BroadcastAudioRing()
ring?.setActive(true, channels: 2, sampleRate: 48_000)
postDarwin(BroadcastNotification.started)
}
override func broadcastFinished() {
ring?.setActive(false)
postDarwin(BroadcastNotification.finished)
ring = nil
}
override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer,
with sampleBufferType: RPSampleBufferType) {
// App/system audio only drop video (the memory hog) and the device mic (the host
// already captures and sends the user's voice).
guard sampleBufferType == .audioApp, let ring else { return }
guard let input = makeInputBuffer(sampleBuffer),
let conv = converter(for: input.format) else { return }
let ratio = outputFormat.sampleRate / input.format.sampleRate
let capacity = AVAudioFrameCount(Double(input.frameLength) * ratio) + 1024
guard let output = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: capacity) else { return }
var supplied = false
var error: NSError?
let status = conv.convert(to: output, error: &error) { _, outStatus in
if supplied { outStatus.pointee = .noDataNow; return nil }
supplied = true
outStatus.pointee = .haveData
return input
}
guard status != .error, output.frameLength > 0,
let mData = output.audioBufferList.pointee.mBuffers.mData else { return }
// Interleaved int16 one buffer of frameLength * channels samples.
let count = Int(output.frameLength) * Int(outputFormat.channelCount)
ring.push(UnsafeBufferPointer(start: mData.assumingMemoryBound(to: Int16.self), count: count))
}
// MARK: - Helpers
/// Wrap the ReplayKit CMSampleBuffer's PCM in an AVAudioPCMBuffer matching its native format.
private func makeInputBuffer(_ sb: CMSampleBuffer) -> AVAudioPCMBuffer? {
guard let fmtDesc = CMSampleBufferGetFormatDescription(sb),
var asbd = CMAudioFormatDescriptionGetStreamBasicDescription(fmtDesc)?.pointee,
let fmt = AVAudioFormat(streamDescription: &asbd) else { return nil }
let frames = AVAudioFrameCount(CMSampleBufferGetNumSamples(sb))
guard frames > 0, let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: frames) else { return nil }
buf.frameLength = frames
let status = CMSampleBufferCopyPCMDataIntoAudioBufferList(
sb, at: 0, frameCount: Int32(frames), into: buf.mutableAudioBufferList)
return status == noErr ? buf : nil
}
/// Reuse the converter while the input format is stable; rebuild if ReplayKit changes it.
private func converter(for inFmt: AVAudioFormat) -> AVAudioConverter? {
if let converter, inputFormat == inFmt { return converter }
inputFormat = inFmt
converter = AVAudioConverter(from: inFmt, to: outputFormat)
return converter
}
private func postDarwin(_ name: String) {
CFNotificationCenterPostNotification(
CFNotificationCenterGetDarwinNotifyCenter(),
CFNotificationName(name as CFString), nil, nil, true)
}
}

View File

@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.application-groups</key>
<array>
<string>group.cat.voice.VoiceCat</string>
</array>
</dict>
</plist>

View File

@@ -13,7 +13,6 @@
BBBB00000000000000000033 /* AudioSessionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001A /* AudioSessionManager.swift */; };
BBBB00000000000000000034 /* ServerListStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001B /* ServerListStore.swift */; };
BBBB00000000000000000035 /* SavedServer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001C /* SavedServer.swift */; };
BBBB00000000000000000036 /* BroadcastCredentials.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001D /* BroadcastCredentials.swift */; };
BBBB00000000000000000037 /* ServerListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001E /* ServerListView.swift */; };
BBBB00000000000000000038 /* AddServerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001F /* AddServerView.swift */; };
BBBB00000000000000000039 /* ServerIdentityView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB00000000000000000020 /* ServerIdentityView.swift */; };
@@ -33,8 +32,31 @@
BBBB00000000000000000047 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002E /* SettingsView.swift */; };
BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002F /* IOSAudioRouter.swift */; };
BBBB00000000000000000048 /* VoiceCatCore in Frameworks */ = {isa = PBXBuildFile; productRef = BBBB0000000000000000004A /* VoiceCatCore */; };
CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000002 /* BroadcastAudioPump.swift */; };
CCCC00000000000000000011 /* BroadcastAudioRing.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000001 /* BroadcastAudioRing.swift */; };
CCCC00000000000000000012 /* SampleHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000003 /* SampleHandler.swift */; };
CCCC00000000000000000013 /* BroadcastAudioRing.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000001 /* BroadcastAudioRing.swift */; };
CCCC00000000000000000014 /* VoiceCatBroadcast.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000006 /* VoiceCatBroadcast.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
CCCC00000000000000000036 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = BBBB00000000000000000001 /* Project object */;
proxyType = 1;
remoteGlobalIDString = CCCC00000000000000000030 /* VoiceCatBroadcast */;
remoteInfo = VoiceCatBroadcast;
};
/* End PBXContainerItemProxy section */
/* Begin PBXTargetDependency section */
CCCC00000000000000000035 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = CCCC00000000000000000030 /* VoiceCatBroadcast */;
targetProxy = CCCC00000000000000000036 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
/* Begin PBXFileReference section */
BBBB00000000000000000012 /* VoiceCatiOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VoiceCatiOS.app; sourceTree = BUILT_PRODUCTS_DIR; };
BBBB00000000000000000015 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
@@ -45,7 +67,6 @@
BBBB0000000000000000001A /* AudioSessionManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioSessionManager.swift; sourceTree = "<group>"; };
BBBB0000000000000000001B /* ServerListStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerListStore.swift; sourceTree = "<group>"; };
BBBB0000000000000000001C /* SavedServer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SavedServer.swift; sourceTree = "<group>"; };
BBBB0000000000000000001D /* BroadcastCredentials.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastCredentials.swift; sourceTree = "<group>"; };
BBBB0000000000000000001E /* ServerListView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerListView.swift; sourceTree = "<group>"; };
BBBB0000000000000000001F /* AddServerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddServerView.swift; sourceTree = "<group>"; };
BBBB00000000000000000020 /* ServerIdentityView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerIdentityView.swift; sourceTree = "<group>"; };
@@ -64,8 +85,28 @@
BBBB0000000000000000002D /* AccountsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountsView.swift; sourceTree = "<group>"; };
BBBB0000000000000000002E /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = "<group>"; };
BBBB0000000000000000002F /* IOSAudioRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSAudioRouter.swift; sourceTree = "<group>"; };
CCCC00000000000000000001 /* BroadcastAudioRing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioRing.swift; sourceTree = "<group>"; };
CCCC00000000000000000002 /* BroadcastAudioPump.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioPump.swift; sourceTree = "<group>"; };
CCCC00000000000000000003 /* SampleHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SampleHandler.swift; sourceTree = "<group>"; };
CCCC00000000000000000004 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
CCCC00000000000000000005 /* VoiceCatBroadcast.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = VoiceCatBroadcast.entitlements; sourceTree = "<group>"; };
CCCC00000000000000000006 /* VoiceCatBroadcast.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = VoiceCatBroadcast.appex; sourceTree = BUILT_PRODUCTS_DIR; };
/* End PBXFileReference section */
/* Begin PBXCopyFilesBuildPhase section */
CCCC00000000000000000037 /* Embed Foundation Extensions */ = {
isa = PBXCopyFilesBuildPhase;
buildActionMask = 2147483647;
dstPath = "";
dstSubfolderSpec = 13;
files = (
CCCC00000000000000000014 /* VoiceCatBroadcast.appex in Embed Foundation Extensions */,
);
name = "Embed Foundation Extensions";
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFrameworksBuildPhase section */
BBBB00000000000000000011 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
@@ -82,6 +123,8 @@
isa = PBXGroup;
children = (
BBBB00000000000000000003 /* VoiceCatiOS */,
CCCC00000000000000000021 /* VoiceCatBroadcast */,
CCCC00000000000000000020 /* Shared */,
BBBB00000000000000000007 /* Products */,
);
sourceTree = "<group>";
@@ -98,12 +141,30 @@
BBBB0000000000000000002F /* IOSAudioRouter.swift */,
BBBB0000000000000000001B /* ServerListStore.swift */,
BBBB0000000000000000001C /* SavedServer.swift */,
BBBB0000000000000000001D /* BroadcastCredentials.swift */,
CCCC00000000000000000002 /* BroadcastAudioPump.swift */,
BBBB00000000000000000006 /* Views */,
);
path = VoiceCatiOS;
sourceTree = "<group>";
};
CCCC00000000000000000020 /* Shared */ = {
isa = PBXGroup;
children = (
CCCC00000000000000000001 /* BroadcastAudioRing.swift */,
);
path = Shared;
sourceTree = "<group>";
};
CCCC00000000000000000021 /* VoiceCatBroadcast */ = {
isa = PBXGroup;
children = (
CCCC00000000000000000003 /* SampleHandler.swift */,
CCCC00000000000000000004 /* Info.plist */,
CCCC00000000000000000005 /* VoiceCatBroadcast.entitlements */,
);
path = VoiceCatBroadcast;
sourceTree = "<group>";
};
BBBB00000000000000000006 /* Views */ = {
isa = PBXGroup;
children = (
@@ -132,6 +193,7 @@
isa = PBXGroup;
children = (
BBBB00000000000000000012 /* VoiceCatiOS.app */,
CCCC00000000000000000006 /* VoiceCatBroadcast.appex */,
);
name = Products;
sourceTree = "<group>";
@@ -146,10 +208,12 @@
BBBB0000000000000000000F /* Sources */,
BBBB00000000000000000010 /* Resources */,
BBBB00000000000000000011 /* Frameworks */,
CCCC00000000000000000037 /* Embed Foundation Extensions */,
);
buildRules = (
);
dependencies = (
CCCC00000000000000000035 /* PBXTargetDependency */,
);
name = VoiceCatiOS;
packageProductDependencies = (
@@ -159,6 +223,21 @@
productReference = BBBB00000000000000000012 /* VoiceCatiOS.app */;
productType = "com.apple.product-type.application";
};
CCCC00000000000000000030 /* VoiceCatBroadcast */ = {
isa = PBXNativeTarget;
buildConfigurationList = CCCC00000000000000000032 /* Build configuration list for PBXNativeTarget "VoiceCatBroadcast" */;
buildPhases = (
CCCC00000000000000000031 /* Sources */,
);
buildRules = (
);
dependencies = (
);
name = VoiceCatBroadcast;
productName = VoiceCatBroadcast;
productReference = CCCC00000000000000000006 /* VoiceCatBroadcast.appex */;
productType = "com.apple.product-type.app-extension";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
@@ -186,6 +265,7 @@
projectRoot = "";
targets = (
BBBB00000000000000000008 /* VoiceCatiOS */,
CCCC00000000000000000030 /* VoiceCatBroadcast */,
);
};
/* End PBXProject section */
@@ -212,7 +292,8 @@
BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */,
BBBB00000000000000000034 /* ServerListStore.swift in Sources */,
BBBB00000000000000000035 /* SavedServer.swift in Sources */,
BBBB00000000000000000036 /* BroadcastCredentials.swift in Sources */,
CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */,
CCCC00000000000000000011 /* BroadcastAudioRing.swift in Sources */,
BBBB00000000000000000037 /* ServerListView.swift in Sources */,
BBBB00000000000000000038 /* AddServerView.swift in Sources */,
BBBB00000000000000000039 /* ServerIdentityView.swift in Sources */,
@@ -233,6 +314,15 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
CCCC00000000000000000031 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
CCCC00000000000000000012 /* SampleHandler.swift in Sources */,
CCCC00000000000000000013 /* BroadcastAudioRing.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin XCBuildConfiguration section */
@@ -408,6 +498,57 @@
};
name = Release;
};
CCCC00000000000000000033 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_ENTITLEMENTS = VoiceCatBroadcast/VoiceCatBroadcast.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = FJV8L966W4;
GENERATE_INFOPLIST_FILE = NO;
INFOPLIST_FILE = VoiceCatBroadcast/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 18.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 0.0.1;
PRODUCT_BUNDLE_IDENTIFIER = cat.voice.VoiceCatiOS.broadcast;
PRODUCT_NAME = "$(TARGET_NAME)";
SKIP_INSTALL = YES;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.9;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
CCCC00000000000000000034 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_ENTITLEMENTS = VoiceCatBroadcast/VoiceCatBroadcast.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = FJV8L966W4;
GENERATE_INFOPLIST_FILE = NO;
INFOPLIST_FILE = VoiceCatBroadcast/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 18.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 0.0.1;
PRODUCT_BUNDLE_IDENTIFIER = cat.voice.VoiceCatiOS.broadcast;
PRODUCT_NAME = "$(TARGET_NAME)";
SKIP_INSTALL = YES;
SWIFT_COMPILATION_MODE = wholemodule;
SWIFT_OPTIMIZATION_LEVEL = "-O";
SWIFT_VERSION = 5.9;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
@@ -429,6 +570,15 @@
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
CCCC00000000000000000032 /* Build configuration list for PBXNativeTarget "VoiceCatBroadcast" */ = {
isa = XCConfigurationList;
buildConfigurations = (
CCCC00000000000000000033 /* Debug */,
CCCC00000000000000000034 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
/* Begin XCLocalSwiftPackageReference section */

View File

@@ -153,7 +153,6 @@ final class AppState {
guard let client = connectingClient else { break }
let perms = client.getPermissions()
let newSession = SessionState(client: client, selfUserId: ev.userId, permissions: perms)
ServerListStore.shared.writeBroadcastCredentials(server: server, nickname: nil)
connectingClient = nil
isConnecting = false
connectStatus = ""

View File

@@ -0,0 +1,159 @@
import Foundation
// BroadcastAudioPump host side of iOS screen-audio sharing (docs/voice.md §9, iOS detail).
//
// Drains the App Group shared-memory ring written by the broadcast upload extension and hands
// whole 20 ms frames to a feed closure (which calls VoiceCatClient.feedPcm). The host app owns
// the SCREEN_AUDIO stream, so screen audio appears as a second stream of the SAME user exactly
// like the Windows/macOS desktop-audio share, and a single session (no separate connection).
//
// It reacts to the extension's Darwin notifications for prompt start/stop, and the drain timer
// also watches the ring's active flag as a safety net if a notification is missed. The ring
// always carries stereo; we downmix to mono when the stream's effective config is mono.
//
// Not @MainActor: the drain runs on a background queue (feedPcm is thread-safe). The start/stop
// callbacks are dispatched to main so they can drive @MainActor SessionState.
final class BroadcastAudioPump {
/// The host should start the SCREEN_AUDIO stream (a broadcast became active).
var onBroadcastStarted: (() -> Void)?
/// The host should stop the SCREEN_AUDIO stream (the broadcast ended).
var onBroadcastFinished: (() -> Void)?
private static let frameSamplesPerChannel = 960 // 20 ms @ 48 kHz
private let queue = DispatchQueue(label: "cat.voice.broadcast.pump")
private var ring: BroadcastAudioRing?
private var timer: DispatchSourceTimer?
private var feed: ((UnsafePointer<Int16>, Int, UInt32) -> Void)?
private var streamChannels = 1
private var ringChannels = 2
private var pending: [Int16] = [] // interleaved at ringChannels width
private var scratch = [Int16](repeating: 0, count: 8192)
private var started = false
// MARK: - Lifecycle (host connect/disconnect)
func start() {
guard !started else { return }
ring = try? BroadcastAudioRing()
started = true
registerDarwin()
// Host reconnected while a broadcast is still running pick it up.
if ring?.isActive == true { onBroadcastStarted?() }
}
func stop() {
guard started else { return }
started = false
unregisterDarwin()
endFeeding()
ring = nil
}
// MARK: - Feeding (driven by the host once the stream is live)
/// Begin draining the ring into `feed`. Called after the SCREEN_AUDIO stream's
/// `.streamStarted` event, when its effective channel count is known.
func beginFeeding(streamChannels: UInt32,
feed: @escaping (UnsafePointer<Int16>, Int, UInt32) -> Void) {
queue.async {
self.streamChannels = max(1, min(2, Int(streamChannels)))
self.ringChannels = max(1, Int(self.ring?.channels ?? 2))
self.feed = feed
self.pending.removeAll(keepingCapacity: true)
self.ring?.drainStale() // discard pre-roll buffered before we were ready
self.startTimer()
}
}
func endFeeding() {
queue.async {
self.timer?.cancel()
self.timer = nil
self.feed = nil
self.pending.removeAll(keepingCapacity: true)
}
}
// MARK: - Drain
private func startTimer() {
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now(), repeating: .milliseconds(10), leeway: .milliseconds(2))
t.setEventHandler { [weak self] in self?.drain() }
timer = t
t.resume()
}
private func drain() {
guard let ring, let feed else { return }
// Safety net: broadcast ended but we missed the Darwin note.
if !ring.isActive {
DispatchQueue.main.async { [weak self] in self?.onBroadcastFinished?() }
return
}
while true {
let got = scratch.withUnsafeMutableBufferPointer { ring.read(into: $0) }
if got == 0 { break }
pending.append(contentsOf: scratch[0..<got])
if got < scratch.count { break }
}
emitFrames(feed)
}
private func emitFrames(_ feed: (UnsafePointer<Int16>, Int, UInt32) -> Void) {
let n = Self.frameSamplesPerChannel
let rc = ringChannels
let sc = streamChannels
let inFrame = n * rc
while pending.count >= inFrame {
if sc == rc {
pending.withUnsafeBufferPointer { feed($0.baseAddress!, n, UInt32(sc)) }
} else if sc == 1 && rc == 2 {
var mono = [Int16](repeating: 0, count: n)
pending.withUnsafeBufferPointer { buf in
let p = buf.baseAddress!
for i in 0..<n { mono[i] = Int16((Int(p[i * 2]) + Int(p[i * 2 + 1])) / 2) }
}
mono.withUnsafeBufferPointer { feed($0.baseAddress!, n, 1) }
} else if sc == 2 && rc == 1 {
var stereo = [Int16](repeating: 0, count: n * 2)
pending.withUnsafeBufferPointer { buf in
let p = buf.baseAddress!
for i in 0..<n { stereo[i * 2] = p[i]; stereo[i * 2 + 1] = p[i] }
}
stereo.withUnsafeBufferPointer { feed($0.baseAddress!, n, 2) }
}
pending.removeFirst(inFrame)
}
}
// MARK: - Darwin notifications
private func registerDarwin() {
let observer = Unmanaged.passUnretained(self).toOpaque()
let center = CFNotificationCenterGetDarwinNotifyCenter()
let callback: CFNotificationCallback = { _, observer, name, _, _ in
guard let observer, let name else { return }
let pump = Unmanaged<BroadcastAudioPump>.fromOpaque(observer).takeUnretainedValue()
let raw = name.rawValue as String
DispatchQueue.main.async {
if raw == BroadcastNotification.started { pump.onBroadcastStarted?() }
else if raw == BroadcastNotification.finished { pump.onBroadcastFinished?() }
}
}
CFNotificationCenterAddObserver(center, observer, callback,
BroadcastNotification.started as CFString, nil, .deliverImmediately)
CFNotificationCenterAddObserver(center, observer, callback,
BroadcastNotification.finished as CFString, nil, .deliverImmediately)
}
private func unregisterDarwin() {
CFNotificationCenterRemoveEveryObserver(
CFNotificationCenterGetDarwinNotifyCenter(),
Unmanaged.passUnretained(self).toOpaque())
}
deinit { if started { unregisterDarwin() } }
}

View File

@@ -1,21 +0,0 @@
import Foundation
struct BroadcastCredentials: Codable {
var host: String
var port: Int
var authMode: String
var username: String
var tofuPinsPath: String
static func loadFromAppGroup() -> BroadcastCredentials? {
let groupId = "group.cat.voice.VoiceCat"
guard let container = FileManager.default.containerURL(
forSecurityApplicationGroupIdentifier: groupId)
else { return nil }
let url = container
.appendingPathComponent("voicecat", isDirectory: true)
.appendingPathComponent("broadcast_credentials.json")
guard let data = try? Data(contentsOf: url) else { return nil }
return try? JSONDecoder().decode(BroadcastCredentials.self, from: data)
}
}

View File

@@ -89,18 +89,4 @@ final class ServerListStore {
SecItemDelete(query as CFDictionary)
}
// MARK: - Broadcast credentials (shared with ReplayKit extension)
func writeBroadcastCredentials(server: SavedServer, nickname: String?) {
let creds = BroadcastCredentials(
host: server.host,
port: Int(server.port),
authMode: server.authMode.rawValue,
username: server.authMode == .password ? server.savedUsername : (nickname ?? ""),
tofuPinsPath: tofuStorePath
)
guard let data = try? JSONEncoder().encode(creds) else { return }
let url = voicecatDir.appendingPathComponent("broadcast_credentials.json")
try? data.write(to: url, options: .atomic)
}
}

View File

@@ -29,6 +29,8 @@ struct VoiceState {
var level: Float = 0.0
var currentDeviceId: String?
var localStreamId: UInt32 = 0
var screenSharing = false
var screenStreamId: UInt32 = 0
}
// MARK: - SessionState
@@ -49,6 +51,10 @@ final class SessionState {
var accounts: [Account] = []
var devices: [Device] = []
/// Host side of iOS screen-audio sharing drains the broadcast extension's App Group ring
/// and feeds the SCREEN_AUDIO stream this session owns. See BroadcastAudioPump.
private let broadcastPump = BroadcastAudioPump()
init(client: VoiceCatClient, selfUserId: UInt32, permissions: Permissions) {
self.client = client
self.selfUserId = selfUserId
@@ -64,9 +70,13 @@ final class SessionState {
client.onLevel = { [weak self] _, rms in
Task { @MainActor [weak self] in self?.voiceState.level = rms }
}
broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() }
broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() }
broadcastPump.start()
}
deinit {
broadcastPump.stop()
MainActor.assumeIsolated {
AudioSessionManager.shared.client = nil
}
@@ -100,6 +110,19 @@ final class SessionState {
let who = users.first(where: { $0.id == ev.userId })?.nickname ?? "user \(ev.userId)"
addActivity(talking ? "\(who) started talking" : "\(who) stopped talking")
case .streamStarted:
// Our own SCREEN_AUDIO stream is live begin draining the broadcast ring into it,
// in the stream's effective channel mode (downmix to mono if the channel is mono).
if ev.userId == selfUserId && ev.streamId == voiceState.screenStreamId {
let sid = voiceState.screenStreamId
let (r, cfg) = client.getStreamAudioConfig(userId: selfUserId, streamId: sid)
let channels: UInt32 = (r == .ok && cfg?.stereo == true) ? 2 : 1
let c = client
broadcastPump.beginFeeding(streamChannels: channels) { pcm, samples, ch in
c.feedPcm(streamId: sid, pcm: pcm, samplesPerChannel: samples, channels: ch)
}
addActivity("Sharing screen audio (\(channels == 2 ? "stereo" : "mono"))")
break
}
// A remote user started a stream ensure the audio session is active so we can
// hear them even if we haven't joined voice ourselves.
if ev.userId != selfUserId {
@@ -236,6 +259,41 @@ final class SessionState {
// disconnecting from the server (see AppState.disconnect / .disconnected event).
}
// MARK: - Screen audio share
/// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream;
/// feeding begins on the resulting `.streamStarted` event (see handleEvent). The actual
/// system-audio capture happens in the ReplayKit upload extension (a separate process).
private func startScreenShare() {
guard voiceState.screenStreamId == 0 else { return }
guard currentChannelId != 0 else {
addActivity("Screen audio ignored — join a channel first")
return
}
let (result, streamId) = client.startStream(
StreamDescriptor(kind: .screenAudio, deviceId: nil, label: "Screen audio"))
if result == .ok {
voiceState.screenStreamId = streamId
voiceState.screenSharing = true
addActivity("Screen audio share starting…")
} else {
addActivity("Failed to start screen audio: \(result.description)")
}
}
/// Called when the broadcast ends (or on disconnect). Stops feeding and the stream.
private func stopScreenShare() {
broadcastPump.endFeeding()
if voiceState.screenStreamId != 0 {
client.stopStream(voiceState.screenStreamId)
voiceState.screenStreamId = 0
}
if voiceState.screenSharing {
voiceState.screenSharing = false
addActivity("Stopped sharing screen audio")
}
}
func setMute(_ muted: Bool, deafened: Bool) {
client.setSelfMute(micMuted: muted, deafened: deafened)
voiceState.selfMuted = muted

View File

@@ -1,5 +1,6 @@
import SwiftUI
import VoiceCatCore
import ReplayKit
struct VoiceControlsView: View {
@Bindable var session: SessionState
@@ -58,6 +59,14 @@ struct VoiceControlsView: View {
.disabled(!session.voiceState.micActive)
.accessibilityLabel(session.voiceState.selfDeafened ? "Undeafen" : "Deafen")
// Share screen audio (ReplayKit system broadcast picker). The picker launches the
// broadcast upload extension; the host's BroadcastAudioPump then owns + feeds the
// SCREEN_AUDIO stream. Tinted while sharing.
BroadcastPickerButton(isSharing: session.voiceState.screenSharing)
.frame(width: 32, height: 32)
.accessibilityLabel(session.voiceState.screenSharing
? "Stop sharing screen audio" : "Share screen audio")
// Disconnect
Button(role: .destructive) {
session.stopMicStream()
@@ -108,6 +117,26 @@ private struct PTTButton: View {
}
}
// MARK: - Broadcast picker
/// Wraps `RPSystemBroadcastPickerView` (which contains its own button) and points it at our
/// broadcast upload extension. Tapping it shows the system broadcast picker; the user starts the
/// broadcast and our extension launches.
private struct BroadcastPickerButton: UIViewRepresentable {
let isSharing: Bool
func makeUIView(context: Context) -> RPSystemBroadcastPickerView {
let picker = RPSystemBroadcastPickerView(frame: CGRect(x: 0, y: 0, width: 32, height: 32))
picker.preferredExtension = "cat.voice.VoiceCatiOS.broadcast"
picker.showsMicrophoneButton = false
return picker
}
func updateUIView(_ uiView: RPSystemBroadcastPickerView, context: Context) {
uiView.tintColor = isSharing ? .systemGreen : .label
}
}
// MARK: - Level Meter
private struct LevelMeterView: View {