Talon/chat_grid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add docker setup and switch voice chat backend to use livekit

Browse Source
This commit is contained in:
Talon
2026-03-11 16:06:41 +01:00
parent b051a0a851
commit f54fff5fb5
24 changed files with 1362 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
client/src/main.ts
View File

@@ -284,8 +284,6 @@ let heartbeatAwaitingPong = false;
let reconnectInFlight = false;
let activeServerInstanceId: string | null = null;
let reloadScheduledForVersionMismatch = false;
let peerNegotiationReady = false;
let pendingSignalMessages: Array<Extract<IncomingMessage, { type: 'signal' }>> = [];
let peerListenGainByNickname = settings.loadPeerListenGains();
let audioLayers: AudioLayerState = {
voice: true,
@@ -322,10 +320,6 @@ const signaling = new SignalingClient(signalingUrl, handleSignalingStatus);
const peerManager = new PeerManager(
audio,
(targetId, payload) => {
signaling.send({ type: 'signal', targetId, ...payload });
},
() => mediaSession.getOutboundStream(),
updateStatus,
);
const mediaSession = new MediaSession({
@@ -1277,7 +1271,7 @@ async function checkMicPermission(): Promise<boolean> {
/** Starts local microphone capture and rebuilds the outbound track pipeline. */
async function setupLocalMedia(audioDeviceId = ''): Promise<void> {
await mediaSession.setupLocalMedia(audioDeviceId);
applyVoiceSendPermission();
authController.applyVoiceSendPermission();
}
/** Runs a short RMS sample to estimate and apply a usable microphone input gain. */
@@ -1497,25 +1491,20 @@ function disconnect(): void {
lastSubscriptionRefreshTileY = Math.round(state.player.y);
stopTeleportLoopAudio();
activeTeleport = null;
peerNegotiationReady = false;
pendingSignalMessages = [];
itemInteractionController.reset();
itemBehaviorRegistry.cleanup();
}
/** Starts peer negotiation only after welcome + media setup sequencing is complete. */
async function activatePeerNegotiation(): Promise<void> {
if (!state.running) return;
if (peerNegotiationReady) return;
peerNegotiationReady = true;
for (const peer of state.peers.values()) {
await peerManager.createOrGetPeer(peer.id, true, peer);
}
if (pendingSignalMessages.length === 0) return;
const queued = pendingSignalMessages;
pendingSignalMessages = [];
for (const signal of queued) {
await onAppMessage(signal);
/** Connects to the LiveKit room and ensures peers exist for roster members. */
async function connectLiveKit(url: string, token: string): Promise<void> {
try {
await peerManager.connectToRoom(url, token);
for (const peer of state.peers.values()) {
peerManager.ensurePeer(peer.id, peer);
}
} catch (error) {
console.error('LiveKit connect failed:', error);
updateStatus('LiveKit connection failed.');
}
}
@@ -1593,12 +1582,8 @@ const onAppMessage = createOnMessageHandler({
handleAdminUsersList,
handleAdminActionResult,
handleItemTransferTargets,
isPeerNegotiationReady: () => peerNegotiationReady,
enqueuePendingSignal: (message) => {
pendingSignalMessages.push(message);
if (pendingSignalMessages.length > 500) {
pendingSignalMessages.splice(0, pendingSignalMessages.length - 500);
}
connectToLiveKit: (url, token) => {
void connectLiveKit(url, token);
},
});
@@ -1670,14 +1655,12 @@ async function setupMediaAfterAuth(): Promise<void> {
const canProceed = await checkMicPermission();
if (!canProceed) {
setConnectionStatus('Microphone access is required.');
await activatePeerNegotiation();
return;
}
try {
await populateAudioDevices();
if (dom.audioInputSelect.options.length === 0) {
setConnectionStatus('No audio input device found. Open Audio setup or connect a microphone.');
await activatePeerNegotiation();
return;
}
const inputDeviceId = dom.audioInputSelect.value || mediaSession.getPreferredInputDeviceId();
@@ -1685,8 +1668,6 @@ async function setupMediaAfterAuth(): Promise<void> {
} catch (error) {
console.error(error);
setConnectionStatus(describeMediaError(error));
} finally {
await activatePeerNegotiation();
}
}

32
client/src/network/messageHandlers.ts
View File

@@ -32,8 +32,7 @@ type MessageHandlerDeps = {
};
signalingSend: (message: unknown) => void;
peerManager: {
createOrGetPeer: (id: string, initiator: boolean, user: { id: string; nickname: string; x: number; y: number }) => Promise<unknown>;
handleSignal: (message: IncomingMessage) => Promise<{ id: string; nickname: string; x: number; y: number }>;
ensurePeer: (id: string, user: { id: string; nickname: string; x: number; y: number }) => { id: string; nickname: string; x: number; y: number };
setPeerPosition: (id: string, x: number, y: number) => void;
setPeerNickname: (id: string, nickname: string) => void;
removePeer: (id: string) => void;
@@ -77,8 +76,7 @@ type MessageHandlerDeps = {
handleAdminUsersList: (message: Extract<IncomingMessage, { type: 'admin_users_list' }>) => void;
handleAdminActionResult: (message: Extract<IncomingMessage, { type: 'admin_action_result' }>) => void;
handleItemTransferTargets: (message: Extract<IncomingMessage, { type: 'item_transfer_targets' }>) => void;
isPeerNegotiationReady: () => boolean;
enqueuePendingSignal: (message: Extract<IncomingMessage, { type: 'signal' }>) => void;
connectToLiveKit: (url: string, token: string) => void;
};
/**
@@ -153,30 +151,8 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
deps.gameLoop();
break;
case 'signal': {
if (!deps.isPeerNegotiationReady()) {
deps.enqueuePendingSignal(message);
if (!deps.state.peers.has(message.senderId)) {
deps.state.peers.set(message.senderId, {
id: message.senderId,
userId: null,
nickname: deps.sanitizeName(message.senderNickname || 'user...') || 'user...',
x: Number.isFinite(message.x) ? message.x : 20,
y: Number.isFinite(message.y) ? message.y : 20,
});
}
break;
}
const peer = await deps.peerManager.handleSignal(message);
if (!deps.state.peers.has(peer.id)) {
deps.state.peers.set(peer.id, {
id: peer.id,
userId: null,
nickname: deps.sanitizeName(peer.nickname) || 'user...',
x: peer.x,
y: peer.y,
});
}
case 'livekit_token': {
deps.connectToLiveKit(message.url, message.token);
break;
}

16
client/src/network/protocol.ts
View File

@@ -175,15 +175,10 @@ export const authResultSchema = z.object({
.optional(),
});
export const signalMessageSchema = z.object({
type: z.literal('signal'),
senderId: z.string(),
senderNickname: z.string().optional(),
x: z.number().int().optional(),
y: z.number().int().optional(),
targetId: z.string().optional(),
sdp: z.any().optional(),
ice: z.any().optional(),
export const livekitTokenSchema = z.object({
type: z.literal('livekit_token'),
token: z.string(),
url: z.string(),
});
export const updatePositionSchema = z.object({
@@ -368,7 +363,7 @@ export const incomingMessageSchema = z.discriminatedUnion('type', [
authRequiredSchema,
authResultSchema,
welcomeMessageSchema,
signalMessageSchema,
livekitTokenSchema,
updatePositionSchema,
teleportCompleteSchema,
updateNicknameSchema,
@@ -407,7 +402,6 @@ export type OutgoingMessage =
| { type: 'admin_user_ban'; username: string }
| { type: 'admin_user_unban'; username: string }
| { type: 'admin_user_delete'; username: string }
| { type: 'signal'; targetId: string; sdp?: RTCSessionDescriptionInit; ice?: RTCIceCandidateInit }
| { type: 'update_position'; x: number; y: number }
| { type: 'teleport_complete'; x: number; y: number }
| { type: 'update_nickname'; nickname: string }

1
client/src/session/authController.ts
View File

@@ -459,6 +459,7 @@ export function createAuthController(deps: AuthControllerDeps): {
handleAuthResult,
handleAuthPermissions,
applyWelcomeAuth,
applyVoiceSendPermission,
logOutAccount,
};
}

235
client/src/webrtc/peerManager.ts
View File

@@ -1,24 +1,32 @@
import {
Room,
RoomEvent,
Track,
RemoteTrack,
RemoteTrackPublication,
RemoteParticipant,
LocalTrack,
LocalAudioTrack,
type AudioCaptureOptions,
} from 'livekit-client';
import { AudioEngine, type SpatialPeerRuntime } from '../audio/audioEngine';
import type { RemoteUser } from '../network/protocol';
export type PeerRuntime = SpatialPeerRuntime & {
id: string;
pc: RTCPeerConnection;
remoteStream?: MediaStream;
};
type SendSignal = (targetId: string, payload: { sdp?: RTCSessionDescriptionInit; ice?: RTCIceCandidateInit }) => void;
type StatusHandler = (message: string) => void;
export class PeerManager {
private readonly peers = new Map<string, PeerRuntime>();
private outputDeviceId = '';
private room: Room | null = null;
private localTrack: LocalAudioTrack | null = null;
constructor(
private readonly audio: AudioEngine,
private readonly sendSignal: SendSignal,
private readonly getLocalStream: () => MediaStream | null,
private readonly status: StatusHandler,
) {}
@@ -30,127 +38,114 @@ export class PeerManager {
return this.peers.values();
}
async createOrGetPeer(targetId: string, isInitiator: boolean, userData: Partial<RemoteUser>): Promise<PeerRuntime> {
/** Connect to a LiveKit room using the provided token and URL. */
async connectToRoom(url: string, token: string): Promise<void> {
if (this.room) {
await this.room.disconnect();
}
const room = new Room({
audioCaptureDefaults: {
sampleRate: 48000,
channelCount: 2,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
} as AudioCaptureOptions,
audioOutput: {
deviceId: this.outputDeviceId || undefined,
},
publishDefaults: {
audioPreset: {
maxBitrate: 128_000,
},
dtx: false,
red: true,
stopMicTrackOnMute: false,
},
});
room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, publication: RemoteTrackPublication, participant: RemoteParticipant) => {
if (track.kind !== Track.Kind.Audio) return;
void this.handleRemoteTrackSubscribed(participant, track);
});
room.on(RoomEvent.TrackUnsubscribed, (_track: RemoteTrack, _publication: RemoteTrackPublication, participant: RemoteParticipant) => {
const peer = this.peers.get(participant.identity);
if (peer) {
this.audio.cleanupPeerAudio(peer);
peer.remoteStream = undefined;
}
});
room.on(RoomEvent.ParticipantDisconnected, (participant: RemoteParticipant) => {
const peer = this.peers.get(participant.identity);
if (peer) {
this.audio.cleanupPeerAudio(peer);
peer.remoteStream = undefined;
}
});
room.on(RoomEvent.Disconnected, () => {
this.status('LiveKit disconnected.');
});
room.on(RoomEvent.Reconnecting, () => {
this.status('LiveKit reconnecting...');
});
room.on(RoomEvent.Reconnected, () => {
this.status('LiveKit reconnected.');
});
await room.connect(url, token);
this.room = room;
}
/** Ensure a peer entry exists for a given user (called when roster arrives). */
ensurePeer(targetId: string, userData: Partial<RemoteUser>): PeerRuntime {
const existing = this.peers.get(targetId);
if (existing) return existing;
const pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
const peer: PeerRuntime = {
id: targetId,
nickname: userData.nickname ?? 'user...',
x: userData.x ?? 20,
y: userData.y ?? 20,
listenGain: 1,
pc,
};
this.peers.set(targetId, peer);
const stream = this.getLocalStream();
if (stream) {
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
} else {
// Ensure initial offers still negotiate audio receive even before mic setup finishes.
pc.addTransceiver('audio', { direction: 'sendrecv' });
}
pc.onicecandidate = (event) => {
if (event.candidate) {
this.sendSignal(targetId, { ice: event.candidate.toJSON() });
}
};
pc.ontrack = async (event) => {
peer.remoteStream = event.streams[0];
if (this.audio.isVoiceLayerEnabled()) {
await this.audio.attachRemoteStream(peer, event.streams[0], this.outputDeviceId);
} else {
this.audio.cleanupPeerAudio(peer);
}
};
if (isInitiator) {
let offer = await pc.createOffer();
offer = this.tuneOpus(offer);
await pc.setLocalDescription(offer);
this.sendSignal(targetId, { sdp: pc.localDescription ?? undefined });
}
return peer;
}
async handleSignal(data: {
senderId: string;
senderNickname?: string;
x?: number;
y?: number;
sdp?: RTCSessionDescriptionInit;
ice?: RTCIceCandidateInit;
}): Promise<PeerRuntime> {
const peer = await this.createOrGetPeer(data.senderId, false, {
id: data.senderId,
nickname: data.senderNickname,
x: data.x,
y: data.y,
});
if (data.sdp) {
await peer.pc.setRemoteDescription(new RTCSessionDescription(data.sdp));
if (data.sdp.type === 'offer') {
let answer = await peer.pc.createAnswer();
answer = this.tuneOpus(answer);
await peer.pc.setLocalDescription(answer);
this.sendSignal(data.senderId, { sdp: peer.pc.localDescription ?? undefined });
}
}
if (data.ice) {
await peer.pc.addIceCandidate(new RTCIceCandidate(data.ice)).catch(() => undefined);
}
return peer;
}
/** Publish a local audio stream to the LiveKit room. */
async replaceOutgoingTrack(stream: MediaStream): Promise<void> {
const newTrack = stream.getAudioTracks()[0];
if (!newTrack) {
return;
}
for (const peer of this.peers.values()) {
const sender =
peer.pc.getSenders().find((candidate) => candidate.track?.kind === 'audio') ??
peer.pc
.getTransceivers()
.find((transceiver) => transceiver.receiver.track?.kind === 'audio' || transceiver.sender.track?.kind === 'audio')
?.sender;
if (!sender) {
peer.pc.addTrack(newTrack, stream);
await this.renegotiatePeer(peer);
} else {
await sender.replaceTrack(newTrack);
}
}
}
if (!newTrack) return;
/** Re-negotiate one peer connection after adding a new outbound track. */
private async renegotiatePeer(peer: PeerRuntime): Promise<void> {
if (peer.pc.connectionState === 'closed') return;
if (peer.pc.signalingState !== 'stable') return;
try {
let offer = await peer.pc.createOffer();
offer = this.tuneOpus(offer);
await peer.pc.setLocalDescription(offer);
this.sendSignal(peer.id, { sdp: peer.pc.localDescription ?? undefined });
} catch {
// Best-effort renegotiation; transport-level failures recover on subsequent signaling.
if (!this.room) return;
if (this.localTrack) {
// Replace the underlying MediaStreamTrack on the existing LiveKit track.
await this.localTrack.replaceTrack(newTrack);
} else {
const localAudioTrack = new LocalAudioTrack(newTrack, undefined, false);
await this.room.localParticipant.publishTrack(localAudioTrack, {
audioPreset: {
maxBitrate: 128_000,
},
dtx: false,
red: true,
stopMicTrackOnMute: false,
});
this.localTrack = localAudioTrack;
}
}
removePeer(id: string): void {
const peer = this.peers.get(id);
if (!peer) return;
peer.pc.close();
this.audio.cleanupPeerAudio(peer);
this.peers.delete(id);
}
@@ -159,6 +154,11 @@ export class PeerManager {
for (const id of this.peers.keys()) {
this.removePeer(id);
}
if (this.room) {
void this.room.disconnect();
this.room = null;
}
this.localTrack = null;
}
setPeerPosition(id: string, x: number, y: number): void {
@@ -210,24 +210,19 @@ export class PeerManager {
}
}
private tuneOpus(desc: RTCSessionDescriptionInit): RTCSessionDescriptionInit {
if (!desc.sdp) return desc;
const lines = desc.sdp.split('\r\n');
let opusPayload: string | undefined;
for (const line of lines) {
if (line.includes('opus/48000')) {
const match = line.match(/(\d+) opus\/48000/);
if (match) opusPayload = match[1];
}
private async handleRemoteTrackSubscribed(participant: RemoteParticipant, track: RemoteTrack): Promise<void> {
const mediaStreamTrack = track.mediaStreamTrack;
if (!mediaStreamTrack) return;
const stream = new MediaStream([mediaStreamTrack]);
const peer = this.peers.get(participant.identity);
if (!peer) return;
peer.remoteStream = stream;
if (this.audio.isVoiceLayerEnabled()) {
await this.audio.attachRemoteStream(peer, stream, this.outputDeviceId);
} else {
this.audio.cleanupPeerAudio(peer);
}
if (opusPayload) {
for (let index = 0; index < lines.length; index += 1) {
if (lines[index].includes(`a=fmtp:${opusPayload}`)) {
lines[index] += ';maxaveragebitrate=128000;stereo=1;sprop-stereo=1;useinbandfec=1;usedtx=0';
break;
}
}
}
return { ...desc, sdp: lines.join('\r\n') };
}
}