net: add heartbeat reconnect and server restart/version announcements
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
// Maintainer-controlled web client version.
|
// Maintainer-controlled web client version.
|
||||||
// Format: YYYY.MM.DD Rn (example: 2026.02.20 R2)
|
// Format: YYYY.MM.DD Rn (example: 2026.02.20 R2)
|
||||||
window.CHGRID_WEB_VERSION = "2026.02.22 R159";
|
window.CHGRID_WEB_VERSION = "2026.02.22 R160";
|
||||||
// Optional display timezone for timestamps. Falls back to America/Detroit if unset/invalid.
|
// Optional display timezone for timestamps. Falls back to America/Detroit if unset/invalid.
|
||||||
window.CHGRID_TIME_ZONE = "America/Detroit";
|
window.CHGRID_TIME_ZONE = "America/Detroit";
|
||||||
|
|||||||
@@ -77,6 +77,8 @@ const MIC_CALIBRATION_TARGET_RMS = 0.12;
|
|||||||
const MIC_CALIBRATION_ACTIVE_RMS_THRESHOLD = 0.003;
|
const MIC_CALIBRATION_ACTIVE_RMS_THRESHOLD = 0.003;
|
||||||
const MIC_INPUT_GAIN_SCALE_MULTIPLIER = 2;
|
const MIC_INPUT_GAIN_SCALE_MULTIPLIER = 2;
|
||||||
const MIC_INPUT_GAIN_STEP = 0.05;
|
const MIC_INPUT_GAIN_STEP = 0.05;
|
||||||
|
const HEARTBEAT_INTERVAL_MS = 10_000;
|
||||||
|
const HEARTBEAT_TIMEOUT_MS = 25_000;
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
interface Window {
|
interface Window {
|
||||||
@@ -194,6 +196,11 @@ let pendingEscapeDisconnect = false;
|
|||||||
let micGainLoopbackRestoreState: boolean | null = null;
|
let micGainLoopbackRestoreState: boolean | null = null;
|
||||||
let helpViewerLines: string[] = [];
|
let helpViewerLines: string[] = [];
|
||||||
let helpViewerIndex = 0;
|
let helpViewerIndex = 0;
|
||||||
|
let heartbeatTimerId: number | null = null;
|
||||||
|
let heartbeatLastPongAt = 0;
|
||||||
|
let heartbeatNextPingId = -1;
|
||||||
|
let reconnectInFlight = false;
|
||||||
|
let activeServerInstanceId: string | null = null;
|
||||||
let audioLayers: AudioLayerState = {
|
let audioLayers: AudioLayerState = {
|
||||||
voice: true,
|
voice: true,
|
||||||
item: true,
|
item: true,
|
||||||
@@ -1036,6 +1043,51 @@ function restoreLoopbackAfterMicGainEdit(): void {
|
|||||||
micGainLoopbackRestoreState = null;
|
micGainLoopbackRestoreState = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Stops heartbeat timer and clears in-memory heartbeat state. */
|
||||||
|
function stopHeartbeat(): void {
|
||||||
|
if (heartbeatTimerId !== null) {
|
||||||
|
window.clearInterval(heartbeatTimerId);
|
||||||
|
heartbeatTimerId = null;
|
||||||
|
}
|
||||||
|
heartbeatLastPongAt = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sends one heartbeat ping packet using reserved negative ids. */
|
||||||
|
function sendHeartbeatPing(): void {
|
||||||
|
signaling.send({ type: 'ping', clientSentAt: heartbeatNextPingId });
|
||||||
|
heartbeatNextPingId -= 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Starts heartbeat timer for stale-connection detection. */
|
||||||
|
function startHeartbeat(): void {
|
||||||
|
stopHeartbeat();
|
||||||
|
heartbeatLastPongAt = Date.now();
|
||||||
|
sendHeartbeatPing();
|
||||||
|
heartbeatTimerId = window.setInterval(() => {
|
||||||
|
if (!state.running) return;
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - heartbeatLastPongAt > HEARTBEAT_TIMEOUT_MS) {
|
||||||
|
void reconnectAfterHeartbeatTimeout();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sendHeartbeatPing();
|
||||||
|
}, HEARTBEAT_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Performs one reconnect attempt when heartbeat timeout indicates stale signaling. */
|
||||||
|
async function reconnectAfterHeartbeatTimeout(): Promise<void> {
|
||||||
|
if (reconnectInFlight || !state.running) return;
|
||||||
|
reconnectInFlight = true;
|
||||||
|
stopHeartbeat();
|
||||||
|
updateStatus('Connection stale. Reconnecting...');
|
||||||
|
disconnect();
|
||||||
|
try {
|
||||||
|
await connect();
|
||||||
|
} finally {
|
||||||
|
reconnectInFlight = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Builds dependencies shared by connect/disconnect flow helpers. */
|
/** Builds dependencies shared by connect/disconnect flow helpers. */
|
||||||
function getConnectionFlowDeps(): ConnectFlowDeps {
|
function getConnectionFlowDeps(): ConnectFlowDeps {
|
||||||
return {
|
return {
|
||||||
@@ -1055,7 +1107,7 @@ function getConnectionFlowDeps(): ConnectFlowDeps {
|
|||||||
mediaStopLocalMedia: () => stopLocalMedia(),
|
mediaStopLocalMedia: () => stopLocalMedia(),
|
||||||
signalingConnect: (handler) => signaling.connect(handler as (message: IncomingMessage) => Promise<void>),
|
signalingConnect: (handler) => signaling.connect(handler as (message: IncomingMessage) => Promise<void>),
|
||||||
signalingDisconnect: () => signaling.disconnect(),
|
signalingDisconnect: () => signaling.disconnect(),
|
||||||
onMessage: (message) => onMessage(message as IncomingMessage),
|
onMessage: (message) => onSignalingMessage(message as IncomingMessage),
|
||||||
worldGridSize,
|
worldGridSize,
|
||||||
persistPlayerPosition,
|
persistPlayerPosition,
|
||||||
peerManagerCleanupAll: () => peerManager.cleanupAll(),
|
peerManagerCleanupAll: () => peerManager.cleanupAll(),
|
||||||
@@ -1074,12 +1126,13 @@ async function connect(): Promise<void> {
|
|||||||
|
|
||||||
/** Tears down active session state, media, peers, and UI back to pre-connect mode. */
|
/** Tears down active session state, media, peers, and UI back to pre-connect mode. */
|
||||||
function disconnect(): void {
|
function disconnect(): void {
|
||||||
|
stopHeartbeat();
|
||||||
runDisconnectFlow(getConnectionFlowDeps());
|
runDisconnectFlow(getConnectionFlowDeps());
|
||||||
pendingEscapeDisconnect = false;
|
pendingEscapeDisconnect = false;
|
||||||
restoreLoopbackAfterMicGainEdit();
|
restoreLoopbackAfterMicGainEdit();
|
||||||
}
|
}
|
||||||
|
|
||||||
const onMessage = createOnMessageHandler({
|
const onAppMessage = createOnMessageHandler({
|
||||||
getWorldGridSize: () => worldGridSize,
|
getWorldGridSize: () => worldGridSize,
|
||||||
setWorldGridSize: (size) => {
|
setWorldGridSize: (size) => {
|
||||||
worldGridSize = size;
|
worldGridSize = size;
|
||||||
@@ -1131,6 +1184,29 @@ const onMessage = createOnMessageHandler({
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/** Handles signaling packets with heartbeat/restart metadata before app-level dispatch. */
|
||||||
|
async function onSignalingMessage(message: IncomingMessage): Promise<void> {
|
||||||
|
if (message.type === 'pong' && message.clientSentAt < 0) {
|
||||||
|
heartbeatLastPongAt = Date.now();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let restartAnnouncement: string | null = null;
|
||||||
|
if (message.type === 'welcome') {
|
||||||
|
const incomingInstanceId = String(message.serverInfo?.instanceId ?? '').trim() || null;
|
||||||
|
const incomingVersion = String(message.serverInfo?.version ?? '').trim() || 'unknown';
|
||||||
|
if (activeServerInstanceId && incomingInstanceId && activeServerInstanceId !== incomingInstanceId) {
|
||||||
|
restartAnnouncement = `Server restarted, version ${incomingVersion}.`;
|
||||||
|
}
|
||||||
|
activeServerInstanceId = incomingInstanceId;
|
||||||
|
startHeartbeat();
|
||||||
|
}
|
||||||
|
await onAppMessage(message);
|
||||||
|
if (restartAnnouncement) {
|
||||||
|
updateStatus(restartAnnouncement);
|
||||||
|
audio.sfxUiConfirm();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Toggles local microphone track mute state. */
|
/** Toggles local microphone track mute state. */
|
||||||
function toggleMute(): void {
|
function toggleMute(): void {
|
||||||
state.isMuted = !state.isMuted;
|
state.isMuted = !state.isMuted;
|
||||||
|
|||||||
@@ -34,6 +34,12 @@ export const welcomeMessageSchema = z.object({
|
|||||||
gridSize: z.number().int().positive(),
|
gridSize: z.number().int().positive(),
|
||||||
})
|
})
|
||||||
.optional(),
|
.optional(),
|
||||||
|
serverInfo: z
|
||||||
|
.object({
|
||||||
|
instanceId: z.string(),
|
||||||
|
version: z.string().optional(),
|
||||||
|
})
|
||||||
|
.optional(),
|
||||||
uiDefinitions: z
|
uiDefinitions: z
|
||||||
.object({
|
.object({
|
||||||
itemTypeOrder: z.array(z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget'])),
|
itemTypeOrder: z.array(z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget'])),
|
||||||
|
|||||||
@@ -38,6 +38,9 @@ This is a behavior guide for packet semantics beyond raw schemas.
|
|||||||
## Welcome Metadata
|
## Welcome Metadata
|
||||||
|
|
||||||
- `welcome.worldConfig.gridSize`: server-authoritative grid size used by clients for bounds/drawing.
|
- `welcome.worldConfig.gridSize`: server-authoritative grid size used by clients for bounds/drawing.
|
||||||
|
- `welcome.serverInfo`: server process identity/version metadata:
|
||||||
|
- `instanceId`: unique id generated at server startup
|
||||||
|
- `version`: server package version (or `unknown` fallback)
|
||||||
- `welcome.uiDefinitions`: server-provided item UI definitions:
|
- `welcome.uiDefinitions`: server-provided item UI definitions:
|
||||||
- `itemTypeOrder`: add-item menu order
|
- `itemTypeOrder`: add-item menu order
|
||||||
- `itemTypes[].tooltip`: item-level tooltip/help text
|
- `itemTypes[].tooltip`: item-level tooltip/help text
|
||||||
@@ -53,3 +56,11 @@ This is a behavior guide for packet semantics beyond raw schemas.
|
|||||||
- Server is authoritative for all action validation and normalization.
|
- Server is authoritative for all action validation and normalization.
|
||||||
- Client validates incoming packet shapes and applies runtime behavior.
|
- Client validates incoming packet shapes and applies runtime behavior.
|
||||||
- Client-side item edit validation is convenience only; server remains source of truth.
|
- Client-side item edit validation is convenience only; server remains source of truth.
|
||||||
|
|
||||||
|
## Heartbeat/Stale Recovery
|
||||||
|
|
||||||
|
- Client sends automatic heartbeat `ping` packets every 10 seconds while connected.
|
||||||
|
- Heartbeat pings use negative `clientSentAt` ids and are internal (not user-visible ping status).
|
||||||
|
- If no heartbeat `pong` arrives within 25 seconds, client force-disconnects and reconnects.
|
||||||
|
- After reconnect, if `welcome.serverInfo.instanceId` changed, client announces:
|
||||||
|
`Server restarted, version <version>.`
|
||||||
|
|||||||
@@ -8,12 +8,14 @@
|
|||||||
4. Server sends `welcome` with users/items snapshot.
|
4. Server sends `welcome` with users/items snapshot.
|
||||||
5. Client:
|
5. Client:
|
||||||
- applies `welcome.worldConfig.gridSize` for authoritative grid bounds/rendering
|
- applies `welcome.worldConfig.gridSize` for authoritative grid bounds/rendering
|
||||||
|
- records `welcome.serverInfo` (`instanceId`, `version`) for restart detection
|
||||||
- applies `welcome.uiDefinitions` for item menus/properties/options
|
- applies `welcome.uiDefinitions` for item menus/properties/options
|
||||||
- sends initial `update_position`
|
- sends initial `update_position`
|
||||||
- sends initial `update_nickname`
|
- sends initial `update_nickname`
|
||||||
- creates peer runtimes for known users
|
- creates peer runtimes for known users
|
||||||
- syncs item runtimes (`radio`, `emit`)
|
- syncs item runtimes (`radio`, `emit`)
|
||||||
- applies audio layer state
|
- applies audio layer state
|
||||||
|
- starts signaling heartbeat monitor
|
||||||
- starts game loop
|
- starts game loop
|
||||||
|
|
||||||
## Main Loop
|
## Main Loop
|
||||||
@@ -38,12 +40,22 @@ Core incoming message effects:
|
|||||||
- `item_remove`: remove item and cleanup runtimes.
|
- `item_remove`: remove item and cleanup runtimes.
|
||||||
- `item_action_result`: success/error status for actions.
|
- `item_action_result`: success/error status for actions.
|
||||||
- `item_use_sound`: play one-shot spatial sample (world layer gated).
|
- `item_use_sound`: play one-shot spatial sample (world layer gated).
|
||||||
|
- `pong`:
|
||||||
|
- positive `clientSentAt`: user ping response (`P` command)
|
||||||
|
- negative `clientSentAt`: internal heartbeat response
|
||||||
|
|
||||||
|
## Stale Connection Recovery
|
||||||
|
|
||||||
|
- While running, client sends heartbeat `ping` every 10 seconds.
|
||||||
|
- If no heartbeat `pong` is observed for 25 seconds, client auto-reconnects.
|
||||||
|
- If reconnect lands on a different `welcome.serverInfo.instanceId`, client announces server restart/version.
|
||||||
|
|
||||||
## Disconnect/Cleanup
|
## Disconnect/Cleanup
|
||||||
|
|
||||||
On disconnect:
|
On disconnect:
|
||||||
|
|
||||||
- Close signaling.
|
- Close signaling.
|
||||||
|
- Stop heartbeat monitor.
|
||||||
- Stop local media tracks.
|
- Stop local media tracks.
|
||||||
- Cleanup peers and all audio runtimes.
|
- Cleanup peers and all audio runtimes.
|
||||||
- Reset UI/mode state and lists.
|
- Reset UI/mode state and lists.
|
||||||
|
|||||||
@@ -103,6 +103,7 @@ class WelcomePacket(BasePacket):
|
|||||||
items: list[dict] | None = None
|
items: list[dict] | None = None
|
||||||
worldConfig: dict | None = None
|
worldConfig: dict | None = None
|
||||||
uiDefinitions: dict | None = None
|
uiDefinitions: dict | None = None
|
||||||
|
serverInfo: dict | None = None
|
||||||
|
|
||||||
|
|
||||||
class UserLeftPacket(BasePacket):
|
class UserLeftPacket(BasePacket):
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from importlib.metadata import PackageNotFoundError, version as package_version
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import ssl
|
import ssl
|
||||||
@@ -88,6 +89,11 @@ class SignalingServer:
|
|||||||
self.item_service = ItemService(state_file=state_file)
|
self.item_service = ItemService(state_file=state_file)
|
||||||
self.item_last_use_ms: dict[str, int] = {}
|
self.item_last_use_ms: dict[str, int] = {}
|
||||||
self.grid_size = max(1, grid_size)
|
self.grid_size = max(1, grid_size)
|
||||||
|
self.instance_id = str(uuid.uuid4())
|
||||||
|
try:
|
||||||
|
self.server_version = package_version("chgrid-server")
|
||||||
|
except PackageNotFoundError:
|
||||||
|
self.server_version = "unknown"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def items(self) -> dict[str, WorldItem]:
|
def items(self) -> dict[str, WorldItem]:
|
||||||
@@ -263,6 +269,7 @@ class SignalingServer:
|
|||||||
items=[item.model_dump(exclude_none=True) for item in self.items.values()],
|
items=[item.model_dump(exclude_none=True) for item in self.items.values()],
|
||||||
worldConfig={"gridSize": self.grid_size},
|
worldConfig={"gridSize": self.grid_size},
|
||||||
uiDefinitions=self._build_ui_definitions(),
|
uiDefinitions=self._build_ui_definitions(),
|
||||||
|
serverInfo={"instanceId": self.instance_id, "version": self.server_version},
|
||||||
)
|
)
|
||||||
await self._send(client.websocket, packet)
|
await self._send(client.websocket, packet)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user