Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(apple): screen-audio sharing -- macOS ScreenCaptureKit, iOS ReplayKit

Browse Source 
Implement system/desktop audio sharing on the Apple clients, feeding the
existing SCREEN_AUDIO Opus -> AEAD -> UDP path via vc_stream_feed_pcm. No
C++/protocol/codec changes -- the core was already ready (the Windows-only
loopback is #ifdef VOICECAT_HAS_LOOPBACK; off Windows the stream just waits
for fed PCM). Audio only; video is dropped.

macOS (in-process):
- ScreenAudioCapture.swift drives an audio-only SCStream
  (excludesCurrentProcessAudio), converts Float32 -> int16 in the channel's
  mono/stereo mode, and calls feedPcm. Capture starts on the self
  .streamStarted event (effective config known then). Wired into
  MainWindowController.screenAudioClicked().

iOS (forward-to-host, single session):
- VoiceCatBroadcast: a ReplayKit Broadcast Upload Extension consumes
  .audioApp only, resamples to 48kHz int16 stereo (AVAudioConverter), and
  writes a shared App Group SPSC ring (BroadcastAudioRing.swift). It does
  not link libvoicecat.
- Host BroadcastAudioPump drains the ring (reacting to the extension's
  Darwin notifications) and feeds the SCREEN_AUDIO stream it owns, downmixing
  to mono when the channel is mono. Screen audio appears as a second stream
  of the same user; no credentials persisted. UI is RPSystemBroadcastPicker
  View in VoiceControlsView. Removes the speculative BroadcastCredentials.

Docs: voice.md s9, CLAUDE.md status, PROGRESS.md.
This commit is contained in:
Talon
2026-06-21 00:14:31 +02:00
parent 6ab78fa792
commit 8c90e250f0
17 changed files with 985 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj
View File

@@ -13,7 +13,6 @@
BBBB00000000000000000033 /* AudioSessionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001A /* AudioSessionManager.swift */; };
BBBB00000000000000000034 /* ServerListStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001B /* ServerListStore.swift */; };
BBBB00000000000000000035 /* SavedServer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001C /* SavedServer.swift */; };
BBBB00000000000000000036 /* BroadcastCredentials.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001D /* BroadcastCredentials.swift */; };
BBBB00000000000000000037 /* ServerListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001E /* ServerListView.swift */; };
BBBB00000000000000000038 /* AddServerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000001F /* AddServerView.swift */; };
BBBB00000000000000000039 /* ServerIdentityView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB00000000000000000020 /* ServerIdentityView.swift */; };
@@ -33,8 +32,31 @@
BBBB00000000000000000047 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002E /* SettingsView.swift */; };
BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002F /* IOSAudioRouter.swift */; };
BBBB00000000000000000048 /* VoiceCatCore in Frameworks */ = {isa = PBXBuildFile; productRef = BBBB0000000000000000004A /* VoiceCatCore */; };
CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000002 /* BroadcastAudioPump.swift */; };
CCCC00000000000000000011 /* BroadcastAudioRing.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000001 /* BroadcastAudioRing.swift */; };
CCCC00000000000000000012 /* SampleHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000003 /* SampleHandler.swift */; };
CCCC00000000000000000013 /* BroadcastAudioRing.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000001 /* BroadcastAudioRing.swift */; };
CCCC00000000000000000014 /* VoiceCatBroadcast.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000006 /* VoiceCatBroadcast.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
CCCC00000000000000000036 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = BBBB00000000000000000001 /* Project object */;
proxyType = 1;
remoteGlobalIDString = CCCC00000000000000000030 /* VoiceCatBroadcast */;
remoteInfo = VoiceCatBroadcast;
};
/* End PBXContainerItemProxy section */
/* Begin PBXTargetDependency section */
CCCC00000000000000000035 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = CCCC00000000000000000030 /* VoiceCatBroadcast */;
targetProxy = CCCC00000000000000000036 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
/* Begin PBXFileReference section */
BBBB00000000000000000012 /* VoiceCatiOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VoiceCatiOS.app; sourceTree = BUILT_PRODUCTS_DIR; };
BBBB00000000000000000015 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
@@ -45,7 +67,6 @@
BBBB0000000000000000001A /* AudioSessionManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioSessionManager.swift; sourceTree = "<group>"; };
BBBB0000000000000000001B /* ServerListStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerListStore.swift; sourceTree = "<group>"; };
BBBB0000000000000000001C /* SavedServer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SavedServer.swift; sourceTree = "<group>"; };
BBBB0000000000000000001D /* BroadcastCredentials.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastCredentials.swift; sourceTree = "<group>"; };
BBBB0000000000000000001E /* ServerListView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerListView.swift; sourceTree = "<group>"; };
BBBB0000000000000000001F /* AddServerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddServerView.swift; sourceTree = "<group>"; };
BBBB00000000000000000020 /* ServerIdentityView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerIdentityView.swift; sourceTree = "<group>"; };
@@ -64,8 +85,28 @@
BBBB0000000000000000002D /* AccountsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountsView.swift; sourceTree = "<group>"; };
BBBB0000000000000000002E /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = "<group>"; };
BBBB0000000000000000002F /* IOSAudioRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSAudioRouter.swift; sourceTree = "<group>"; };
CCCC00000000000000000001 /* BroadcastAudioRing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioRing.swift; sourceTree = "<group>"; };
CCCC00000000000000000002 /* BroadcastAudioPump.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioPump.swift; sourceTree = "<group>"; };
CCCC00000000000000000003 /* SampleHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SampleHandler.swift; sourceTree = "<group>"; };
CCCC00000000000000000004 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
CCCC00000000000000000005 /* VoiceCatBroadcast.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = VoiceCatBroadcast.entitlements; sourceTree = "<group>"; };
CCCC00000000000000000006 /* VoiceCatBroadcast.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = VoiceCatBroadcast.appex; sourceTree = BUILT_PRODUCTS_DIR; };
/* End PBXFileReference section */
/* Begin PBXCopyFilesBuildPhase section */
CCCC00000000000000000037 /* Embed Foundation Extensions */ = {
isa = PBXCopyFilesBuildPhase;
buildActionMask = 2147483647;
dstPath = "";
dstSubfolderSpec = 13;
files = (
CCCC00000000000000000014 /* VoiceCatBroadcast.appex in Embed Foundation Extensions */,
);
name = "Embed Foundation Extensions";
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFrameworksBuildPhase section */
BBBB00000000000000000011 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
@@ -82,6 +123,8 @@
isa = PBXGroup;
children = (
BBBB00000000000000000003 /* VoiceCatiOS */,
CCCC00000000000000000021 /* VoiceCatBroadcast */,
CCCC00000000000000000020 /* Shared */,
BBBB00000000000000000007 /* Products */,
);
sourceTree = "<group>";
@@ -98,12 +141,30 @@
BBBB0000000000000000002F /* IOSAudioRouter.swift */,
BBBB0000000000000000001B /* ServerListStore.swift */,
BBBB0000000000000000001C /* SavedServer.swift */,
BBBB0000000000000000001D /* BroadcastCredentials.swift */,
CCCC00000000000000000002 /* BroadcastAudioPump.swift */,
BBBB00000000000000000006 /* Views */,
);
path = VoiceCatiOS;
sourceTree = "<group>";
};
CCCC00000000000000000020 /* Shared */ = {
isa = PBXGroup;
children = (
CCCC00000000000000000001 /* BroadcastAudioRing.swift */,
);
path = Shared;
sourceTree = "<group>";
};
CCCC00000000000000000021 /* VoiceCatBroadcast */ = {
isa = PBXGroup;
children = (
CCCC00000000000000000003 /* SampleHandler.swift */,
CCCC00000000000000000004 /* Info.plist */,
CCCC00000000000000000005 /* VoiceCatBroadcast.entitlements */,
);
path = VoiceCatBroadcast;
sourceTree = "<group>";
};
BBBB00000000000000000006 /* Views */ = {
isa = PBXGroup;
children = (
@@ -132,6 +193,7 @@
isa = PBXGroup;
children = (
BBBB00000000000000000012 /* VoiceCatiOS.app */,
CCCC00000000000000000006 /* VoiceCatBroadcast.appex */,
);
name = Products;
sourceTree = "<group>";
@@ -146,10 +208,12 @@
BBBB0000000000000000000F /* Sources */,
BBBB00000000000000000010 /* Resources */,
BBBB00000000000000000011 /* Frameworks */,
CCCC00000000000000000037 /* Embed Foundation Extensions */,
);
buildRules = (
);
dependencies = (
CCCC00000000000000000035 /* PBXTargetDependency */,
);
name = VoiceCatiOS;
packageProductDependencies = (
@@ -159,6 +223,21 @@
productReference = BBBB00000000000000000012 /* VoiceCatiOS.app */;
productType = "com.apple.product-type.application";
};
CCCC00000000000000000030 /* VoiceCatBroadcast */ = {
isa = PBXNativeTarget;
buildConfigurationList = CCCC00000000000000000032 /* Build configuration list for PBXNativeTarget "VoiceCatBroadcast" */;
buildPhases = (
CCCC00000000000000000031 /* Sources */,
);
buildRules = (
);
dependencies = (
);
name = VoiceCatBroadcast;
productName = VoiceCatBroadcast;
productReference = CCCC00000000000000000006 /* VoiceCatBroadcast.appex */;
productType = "com.apple.product-type.app-extension";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
@@ -186,6 +265,7 @@
projectRoot = "";
targets = (
BBBB00000000000000000008 /* VoiceCatiOS */,
CCCC00000000000000000030 /* VoiceCatBroadcast */,
);
};
/* End PBXProject section */
@@ -212,7 +292,8 @@
BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */,
BBBB00000000000000000034 /* ServerListStore.swift in Sources */,
BBBB00000000000000000035 /* SavedServer.swift in Sources */,
BBBB00000000000000000036 /* BroadcastCredentials.swift in Sources */,
CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */,
CCCC00000000000000000011 /* BroadcastAudioRing.swift in Sources */,
BBBB00000000000000000037 /* ServerListView.swift in Sources */,
BBBB00000000000000000038 /* AddServerView.swift in Sources */,
BBBB00000000000000000039 /* ServerIdentityView.swift in Sources */,
@@ -233,6 +314,15 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
CCCC00000000000000000031 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
CCCC00000000000000000012 /* SampleHandler.swift in Sources */,
CCCC00000000000000000013 /* BroadcastAudioRing.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin XCBuildConfiguration section */
@@ -408,6 +498,57 @@
};
name = Release;
};
CCCC00000000000000000033 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_ENTITLEMENTS = VoiceCatBroadcast/VoiceCatBroadcast.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = FJV8L966W4;
GENERATE_INFOPLIST_FILE = NO;
INFOPLIST_FILE = VoiceCatBroadcast/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 18.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 0.0.1;
PRODUCT_BUNDLE_IDENTIFIER = cat.voice.VoiceCatiOS.broadcast;
PRODUCT_NAME = "$(TARGET_NAME)";
SKIP_INSTALL = YES;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.9;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
CCCC00000000000000000034 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_ENTITLEMENTS = VoiceCatBroadcast/VoiceCatBroadcast.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = FJV8L966W4;
GENERATE_INFOPLIST_FILE = NO;
INFOPLIST_FILE = VoiceCatBroadcast/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 18.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 0.0.1;
PRODUCT_BUNDLE_IDENTIFIER = cat.voice.VoiceCatiOS.broadcast;
PRODUCT_NAME = "$(TARGET_NAME)";
SKIP_INSTALL = YES;
SWIFT_COMPILATION_MODE = wholemodule;
SWIFT_OPTIMIZATION_LEVEL = "-O";
SWIFT_VERSION = 5.9;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
@@ -429,6 +570,15 @@
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
CCCC00000000000000000032 /* Build configuration list for PBXNativeTarget "VoiceCatBroadcast" */ = {
isa = XCConfigurationList;
buildConfigurations = (
CCCC00000000000000000033 /* Debug */,
CCCC00000000000000000034 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
/* Begin XCLocalSwiftPackageReference section */