feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit
Implement system/desktop audio sharing on the Apple clients, feeding the existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No C++/protocol/codec changes -- the core was already ready (the Windows-only loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits for fed PCM). Audio only; video is dropped. macOS (in-process): - ScreenAudioCapture.swift drives an audio-only SCStream (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's mono/stereo mode, and calls feedPcm. Capture starts on the self .streamStarted event (effective config known then). Wired into MainWindowController.screenAudioClicked(). iOS (forward-to-host, single session): - VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does not link libvoicecat. - Host BroadcastAudioPump drains the ring (reacting to the extension's Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing to mono when the channel is mono. Screen audio appears as a second stream of the same user; no credentials persisted. UI is RPSystemBroadcastPicker View in VoiceControlsView. Removes the speculative BroadcastCredentials. Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
88
clients/apple/iOS/VoiceCatBroadcast/SampleHandler.swift
Normal file
88
clients/apple/iOS/VoiceCatBroadcast/SampleHandler.swift
Normal file
@@ -0,0 +1,88 @@
|
||||
import ReplayKit
|
||||
import AVFoundation
|
||||
|
||||
// SampleHandler — the ReplayKit broadcast upload extension entry point (docs/voice.md §9, iOS).
|
||||
//
|
||||
// This runs in a SEPARATE process with a ~50 MB memory cap. It captures system/app audio
|
||||
// (`.audioApp`), drops video and mic buffers, converts each chunk to the core's canonical
|
||||
// format (48 kHz, int16, stereo interleaved) with AVAudioConverter, and writes it into the
|
||||
// App Group shared-memory ring. The host app's BroadcastAudioPump drains the ring and feeds the
|
||||
// already-connected VoiceCatClient — so all Opus/AEAD/UDP work happens in the host, and the
|
||||
// extension stays tiny and well inside the memory budget (no libvoicecat here).
|
||||
class SampleHandler: RPBroadcastSampleHandler {
|
||||
|
||||
private var ring: BroadcastAudioRing?
|
||||
private var converter: AVAudioConverter?
|
||||
private var inputFormat: AVAudioFormat?
|
||||
private let outputFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
|
||||
sampleRate: 48_000, channels: 2, interleaved: true)!
|
||||
|
||||
override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) {
|
||||
ring = try? BroadcastAudioRing()
|
||||
ring?.setActive(true, channels: 2, sampleRate: 48_000)
|
||||
postDarwin(BroadcastNotification.started)
|
||||
}
|
||||
|
||||
override func broadcastFinished() {
|
||||
ring?.setActive(false)
|
||||
postDarwin(BroadcastNotification.finished)
|
||||
ring = nil
|
||||
}
|
||||
|
||||
override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer,
|
||||
with sampleBufferType: RPSampleBufferType) {
|
||||
// App/system audio only — drop video (the memory hog) and the device mic (the host
|
||||
// already captures and sends the user's voice).
|
||||
guard sampleBufferType == .audioApp, let ring else { return }
|
||||
guard let input = makeInputBuffer(sampleBuffer),
|
||||
let conv = converter(for: input.format) else { return }
|
||||
|
||||
let ratio = outputFormat.sampleRate / input.format.sampleRate
|
||||
let capacity = AVAudioFrameCount(Double(input.frameLength) * ratio) + 1024
|
||||
guard let output = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: capacity) else { return }
|
||||
|
||||
var supplied = false
|
||||
var error: NSError?
|
||||
let status = conv.convert(to: output, error: &error) { _, outStatus in
|
||||
if supplied { outStatus.pointee = .noDataNow; return nil }
|
||||
supplied = true
|
||||
outStatus.pointee = .haveData
|
||||
return input
|
||||
}
|
||||
guard status != .error, output.frameLength > 0,
|
||||
let mData = output.audioBufferList.pointee.mBuffers.mData else { return }
|
||||
|
||||
// Interleaved int16 → one buffer of frameLength * channels samples.
|
||||
let count = Int(output.frameLength) * Int(outputFormat.channelCount)
|
||||
ring.push(UnsafeBufferPointer(start: mData.assumingMemoryBound(to: Int16.self), count: count))
|
||||
}
|
||||
|
||||
// MARK: - Helpers
|
||||
|
||||
/// Wrap the ReplayKit CMSampleBuffer's PCM in an AVAudioPCMBuffer matching its native format.
|
||||
private func makeInputBuffer(_ sb: CMSampleBuffer) -> AVAudioPCMBuffer? {
|
||||
guard let fmtDesc = CMSampleBufferGetFormatDescription(sb),
|
||||
var asbd = CMAudioFormatDescriptionGetStreamBasicDescription(fmtDesc)?.pointee,
|
||||
let fmt = AVAudioFormat(streamDescription: &asbd) else { return nil }
|
||||
let frames = AVAudioFrameCount(CMSampleBufferGetNumSamples(sb))
|
||||
guard frames > 0, let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: frames) else { return nil }
|
||||
buf.frameLength = frames
|
||||
let status = CMSampleBufferCopyPCMDataIntoAudioBufferList(
|
||||
sb, at: 0, frameCount: Int32(frames), into: buf.mutableAudioBufferList)
|
||||
return status == noErr ? buf : nil
|
||||
}
|
||||
|
||||
/// Reuse the converter while the input format is stable; rebuild if ReplayKit changes it.
|
||||
private func converter(for inFmt: AVAudioFormat) -> AVAudioConverter? {
|
||||
if let converter, inputFormat == inFmt { return converter }
|
||||
inputFormat = inFmt
|
||||
converter = AVAudioConverter(from: inFmt, to: outputFormat)
|
||||
return converter
|
||||
}
|
||||
|
||||
private func postDarwin(_ name: String) {
|
||||
CFNotificationCenterPostNotification(
|
||||
CFNotificationCenterGetDarwinNotifyCenter(),
|
||||
CFNotificationName(name as CFString), nil, nil, true)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user