Talon/chat_grid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add piano item type with realtime play mode and remote notes

Browse Source
This commit is contained in:
Jage9
2026-02-22 23:42:17 -05:00
parent 81c6af6399
commit 1319c044dd
23 changed files with 1014 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client/public/help.json
View File

@@ -84,6 +84,10 @@
{
"keys": "Enter",
"description": "Use item"
},
{
"keys": "Piano mode",
"description": "When using a piano: ASDFGHJKL;' plays C major notes, WETYUOP] plays sharps, Enter/Escape exits"
}
]
},

2
client/public/version.js
View File

@@ -1,5 +1,5 @@
// Maintainer-controlled web client version.
// Format: YYYY.MM.DD Rn (example: 2026.02.20 R2)
window.CHGRID_WEB_VERSION = "2026.02.22 R198";
window.CHGRID_WEB_VERSION = "2026.02.22 R199";
// Optional display timezone for timestamps. Falls back to America/Detroit if unset/invalid.
window.CHGRID_TIME_ZONE = "America/Detroit";

330
client/src/audio/pianoSynth.ts Normal file
View File

@@ -0,0 +1,330 @@
import { resolveSpatialMix } from './spatial';
export const PIANO_INSTRUMENT_OPTIONS = [
'piano',
'electric_piano',
'guitar',
'organ',
'bass',
'violin',
'synth_lead',
'drum_kit',
] as const;
export type PianoInstrumentId = (typeof PIANO_INSTRUMENT_OPTIONS)[number];
type VoiceRuntime = {
gain: GainNode;
panner: StereoPannerNode | null;
oscillators: OscillatorNode[];
releaseSeconds: number;
};
type PianoContext = {
audioCtx: AudioContext;
destination: AudioNode;
};
type PianoSpatialSource = {
x: number;
y: number;
range: number;
};
type InstrumentPreset = {
oscillators: Array<{ type: OscillatorType; detune?: number; gain?: number; ratio?: number }>;
filter?: { type: BiquadFilterType; frequency: number; q?: number };
gain: number;
releaseScale?: number;
};
const PRESETS: Record<Exclude<PianoInstrumentId, 'drum_kit'>, InstrumentPreset> = {
piano: {
oscillators: [
{ type: 'triangle', gain: 1 },
{ type: 'sine', ratio: 2, gain: 0.28 },
],
filter: { type: 'lowpass', frequency: 5200, q: 0.7 },
gain: 0.32,
releaseScale: 0.9,
},
electric_piano: {
oscillators: [
{ type: 'sine', gain: 1 },
{ type: 'triangle', detune: 5, gain: 0.35 },
],
filter: { type: 'lowpass', frequency: 4200, q: 0.8 },
gain: 0.3,
releaseScale: 0.8,
},
guitar: {
oscillators: [
{ type: 'triangle', gain: 1 },
{ type: 'sawtooth', detune: -3, gain: 0.2 },
],
filter: { type: 'lowpass', frequency: 3200, q: 0.9 },
gain: 0.24,
releaseScale: 0.7,
},
organ: {
oscillators: [
{ type: 'square', gain: 0.8 },
{ type: 'sine', ratio: 2, gain: 0.28 },
{ type: 'sine', ratio: 3, gain: 0.2 },
],
filter: { type: 'lowpass', frequency: 6500, q: 0.6 },
gain: 0.18,
releaseScale: 1.4,
},
bass: {
oscillators: [
{ type: 'sawtooth', gain: 0.9 },
{ type: 'square', ratio: 0.5, gain: 0.25 },
],
filter: { type: 'lowpass', frequency: 1500, q: 1.1 },
gain: 0.28,
releaseScale: 0.9,
},
violin: {
oscillators: [
{ type: 'sawtooth', gain: 0.8 },
{ type: 'triangle', detune: 3, gain: 0.35 },
],
filter: { type: 'lowpass', frequency: 3600, q: 1.0 },
gain: 0.24,
releaseScale: 1.5,
},
synth_lead: {
oscillators: [
{ type: 'sawtooth', gain: 0.85 },
{ type: 'square', detune: 6, gain: 0.3 },
],
filter: { type: 'lowpass', frequency: 5400, q: 0.9 },
gain: 0.2,
releaseScale: 1,
},
};
/** Maps 0..100 control values to note attack seconds. */
function attackPercentToSeconds(value: number): number {
const clamped = Math.max(0, Math.min(100, value));
return 0.002 + (clamped / 100) * 0.6;
}
/** Maps 0..100 control values to note decay/release seconds. */
function decayPercentToSeconds(value: number): number {
const clamped = Math.max(0, Math.min(100, value));
return 0.05 + (clamped / 100) * 2.7;
}
/** Converts midi note number to frequency in hertz. */
function midiToFrequency(midi: number): number {
return 440 * Math.pow(2, (midi - 69) / 12);
}
/** Small helper to safely stop audio nodes. */
function safeStop(oscillator: OscillatorNode, when: number): void {
try {
oscillator.stop(when);
} catch {
// Ignore already-stopped oscillators.
}
}
export class PianoSynth {
private readonly voices = new Map<string, VoiceRuntime>();
private readonly drumNoiseBuffers = new WeakMap<AudioContext, AudioBuffer>();
/** Stops and disconnects all active notes. */
stopAll(): void {
for (const key of Array.from(this.voices.keys())) {
this.noteOff(key);
}
}
/** Starts one note for a specific keyboard key id. */
noteOn(
keyId: string,
midi: number,
instrument: PianoInstrumentId,
attackPercent: number,
decayPercent: number,
context: PianoContext,
spatial: PianoSpatialSource,
): void {
if (this.voices.has(keyId)) return;
if (instrument === 'drum_kit') {
this.playDrumHit(keyId, context, spatial, attackPercent, decayPercent);
return;
}
const preset = PRESETS[instrument] ?? PRESETS.piano;
const now = context.audioCtx.currentTime;
const attackSeconds = attackPercentToSeconds(attackPercent);
const decaySeconds = decayPercentToSeconds(decayPercent);
const releaseSeconds = Math.max(0.02, decaySeconds * (preset.releaseScale ?? 1));
const spatialMix = resolveSpatialMix({
dx: spatial.x,
dy: spatial.y,
range: spatial.range,
baseGain: 1,
});
if (!spatialMix || spatialMix.gain <= 0) return;
const voiceGain = context.audioCtx.createGain();
voiceGain.gain.setValueAtTime(0.0001, now);
const peakGain = Math.max(0.0001, preset.gain * spatialMix.gain);
const sustainGain = Math.max(0.0001, peakGain * 0.55);
voiceGain.gain.exponentialRampToValueAtTime(peakGain, now + attackSeconds);
voiceGain.gain.exponentialRampToValueAtTime(sustainGain, now + attackSeconds + decaySeconds * 0.6);
let tailNode: AudioNode = voiceGain;
if (preset.filter) {
const filter = context.audioCtx.createBiquadFilter();
filter.type = preset.filter.type;
filter.frequency.setValueAtTime(preset.filter.frequency, now);
filter.Q.setValueAtTime(preset.filter.q ?? 0.7, now);
voiceGain.connect(filter);
tailNode = filter;
}
let panner: StereoPannerNode | null = null;
if (typeof context.audioCtx.createStereoPanner === 'function') {
panner = context.audioCtx.createStereoPanner();
panner.pan.setValueAtTime(spatialMix.pan, now);
tailNode.connect(panner).connect(context.destination);
} else {
tailNode.connect(context.destination);
}
const frequency = midiToFrequency(midi);
const oscillators: OscillatorNode[] = [];
for (const partial of preset.oscillators) {
const oscillator = context.audioCtx.createOscillator();
oscillator.type = partial.type;
oscillator.frequency.setValueAtTime(frequency * (partial.ratio ?? 1), now);
oscillator.detune.setValueAtTime(partial.detune ?? 0, now);
const oscGain = context.audioCtx.createGain();
oscGain.gain.setValueAtTime(partial.gain ?? 1, now);
oscillator.connect(oscGain).connect(voiceGain);
oscillator.start(now);
oscillators.push(oscillator);
}
this.voices.set(keyId, {
gain: voiceGain,
panner,
oscillators,
releaseSeconds,
});
}
/** Releases one active note tied to a keyboard key id. */
noteOff(keyId: string): void {
const voice = this.voices.get(keyId);
if (!voice) return;
this.voices.delete(keyId);
const now = voice.gain.context.currentTime;
const currentGain = Math.max(0.0001, voice.gain.gain.value);
voice.gain.gain.cancelScheduledValues(now);
voice.gain.gain.setValueAtTime(currentGain, now);
voice.gain.gain.exponentialRampToValueAtTime(0.0001, now + voice.releaseSeconds);
for (const oscillator of voice.oscillators) {
safeStop(oscillator, now + voice.releaseSeconds + 0.02);
}
window.setTimeout(() => {
try {
voice.gain.disconnect();
} catch {
// Ignore stale disconnects.
}
if (voice.panner) {
try {
voice.panner.disconnect();
} catch {
// Ignore stale disconnects.
}
}
}, Math.max(60, Math.round((voice.releaseSeconds + 0.04) * 1000)));
}
/** Plays one synthesized drum hit for drum-kit instrument mode. */
private playDrumHit(
keyId: string,
context: PianoContext,
spatial: PianoSpatialSource,
attackPercent: number,
decayPercent: number,
): void {
const now = context.audioCtx.currentTime;
const spatialMix = resolveSpatialMix({
dx: spatial.x,
dy: spatial.y,
range: spatial.range,
baseGain: 1,
});
if (!spatialMix || spatialMix.gain <= 0) return;
const typeIndex = Math.abs(this.hashKey(keyId)) % 4;
const decaySeconds = 0.03 + decayPercentToSeconds(decayPercent) * 0.45;
const attackSeconds = Math.max(0.001, attackPercentToSeconds(attackPercent) * 0.2);
const gain = context.audioCtx.createGain();
gain.gain.setValueAtTime(0.0001, now);
gain.gain.exponentialRampToValueAtTime(0.22 * spatialMix.gain, now + attackSeconds);
gain.gain.exponentialRampToValueAtTime(0.0001, now + decaySeconds);
let tailNode: AudioNode = gain;
if (typeof context.audioCtx.createStereoPanner === 'function') {
const panner = context.audioCtx.createStereoPanner();
panner.pan.setValueAtTime(spatialMix.pan, now);
tailNode.connect(panner).connect(context.destination);
tailNode = panner;
} else {
tailNode.connect(context.destination);
}
if (typeIndex === 0) {
const kick = context.audioCtx.createOscillator();
kick.type = 'sine';
kick.frequency.setValueAtTime(150, now);
kick.frequency.exponentialRampToValueAtTime(45, now + decaySeconds * 0.85);
kick.connect(gain);
kick.start(now);
safeStop(kick, now + decaySeconds + 0.04);
return;
}
const noise = context.audioCtx.createBufferSource();
noise.buffer = this.getNoiseBuffer(context.audioCtx);
const noiseFilter = context.audioCtx.createBiquadFilter();
noiseFilter.type = typeIndex === 1 ? 'highpass' : typeIndex === 2 ? 'bandpass' : 'lowpass';
noiseFilter.frequency.setValueAtTime(typeIndex === 1 ? 1700 : typeIndex === 2 ? 900 : 1300, now);
noise.connect(noiseFilter).connect(gain);
noise.start(now);
safeStop(noise, now + decaySeconds + 0.03);
}
/** Returns deterministic hash for key ids to map drum voice variants. */
private hashKey(value: string): number {
let out = 0;
for (let index = 0; index < value.length; index += 1) {
out = ((out << 5) - out + value.charCodeAt(index)) | 0;
}
return out;
}
/** Returns or lazily builds short white-noise buffer for percussion synthesis. */
private getNoiseBuffer(audioCtx: AudioContext): AudioBuffer {
const existing = this.drumNoiseBuffers.get(audioCtx);
if (existing) return existing;
const length = Math.max(1, Math.floor(audioCtx.sampleRate * 0.5));
const buffer = audioCtx.createBuffer(1, length, audioCtx.sampleRate);
const data = buffer.getChannelData(0);
for (let index = 0; index < length; index += 1) {
data[index] = Math.random() * 2 - 1;
}
this.drumNoiseBuffers.set(audioCtx, buffer);
return buffer;
}
}

2
client/src/items/itemPropertyEditor.ts
View File

@@ -309,6 +309,8 @@ export function createItemPropertyEditor(deps: EditorDeps): {
propertyKey === 'mediaVolume' ||
propertyKey === 'emitVolume' ||
propertyKey === 'emitRange' ||
propertyKey === 'attack' ||
propertyKey === 'decay' ||
propertyKey === 'sides' ||
propertyKey === 'number'
) {

21
client/src/items/itemRegistry.ts
View File

@@ -45,11 +45,22 @@ const DEFAULT_CLOCK_TIME_ZONE_OPTIONS = [
'UTC',
] as const;
const DEFAULT_ITEM_TYPE_SEQUENCE: ItemType[] = ['clock', 'dice', 'radio_station', 'wheel', 'widget'];
const DEFAULT_ITEM_TYPE_SEQUENCE: ItemType[] = ['clock', 'dice', 'piano', 'radio_station', 'wheel', 'widget'];
const DEFAULT_PIANO_INSTRUMENT_OPTIONS = [
'piano',
'electric_piano',
'guitar',
'organ',
'bass',
'violin',
'synth_lead',
'drum_kit',
] as const;
const DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES: Record<ItemType, string[]> = {
radio_station: ['title', 'streamUrl', 'enabled', 'mediaVolume', 'mediaChannel', 'mediaEffect', 'mediaEffectValue', 'facing', 'emitRange'],
dice: ['title', 'sides', 'number'],
piano: ['title', 'instrument', 'attack', 'decay', 'emitRange'],
wheel: ['title', 'spaces'],
clock: ['title', 'timeZone', 'use24Hour'],
widget: ['title', 'enabled', 'directional', 'facing', 'emitRange', 'emitVolume', 'emitSoundSpeed', 'emitSoundTempo', 'emitEffect', 'emitEffectValue', 'useSound', 'emitSound'],
@@ -58,6 +69,7 @@ const DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES: Record<ItemType, string[]> = {
const DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES: Record<ItemType, Record<string, string | number | boolean>> = {
radio_station: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 20, directional: true, emitSoundSpeed: 50, emitSoundTempo: 50 },
dice: { useSound: 'sounds/roll.ogg', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
piano: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
wheel: { useSound: 'sounds/spin.ogg', emitSound: 'none', useCooldownMs: 4000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
clock: { useSound: 'none', emitSound: 'sounds/clock.ogg', useCooldownMs: 1000, emitRange: 10, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
widget: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
@@ -93,6 +105,7 @@ let itemTypeSequence: ItemType[] = [...DEFAULT_ITEM_TYPE_SEQUENCE];
let itemTypeLabels: Record<ItemType, string> = {
radio_station: 'radio',
dice: 'dice',
piano: 'piano',
wheel: 'wheel',
clock: 'clock',
widget: 'widget',
@@ -101,6 +114,7 @@ let itemTypeTooltips: Partial<Record<ItemType, string>> = {};
let itemTypeEditableProperties: Record<ItemType, string[]> = {
radio_station: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.radio_station],
dice: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.dice],
piano: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.piano],
wheel: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.wheel],
clock: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.clock],
widget: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.widget],
@@ -108,6 +122,7 @@ let itemTypeEditableProperties: Record<ItemType, string[]> = {
let itemTypeGlobalProperties: Record<ItemType, Record<string, string | number | boolean>> = {
radio_station: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.radio_station },
dice: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.dice },
piano: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.piano },
wheel: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.wheel },
clock: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.clock },
widget: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.widget },
@@ -116,6 +131,7 @@ let optionItemPropertyValues: Partial<Record<string, string[]>> = {
mediaEffect: EFFECT_SEQUENCE.map((effect) => effect.id),
emitEffect: EFFECT_SEQUENCE.map((effect) => effect.id),
mediaChannel: [...RADIO_CHANNEL_OPTIONS],
instrument: [...DEFAULT_PIANO_INSTRUMENT_OPTIONS],
timeZone: [...DEFAULT_CLOCK_TIME_ZONE_OPTIONS],
};
let itemTypePropertyMetadata: Partial<Record<ItemType, Record<string, ItemPropertyMetadata>>> = {};
@@ -221,6 +237,9 @@ export function itemPropertyLabel(key: string): string {
if (key === 'mediaEffectValue') return 'media effect value';
if (key === 'emitEffect') return 'emit effect';
if (key === 'emitEffectValue') return 'emit effect value';
if (key === 'instrument') return 'instrument';
if (key === 'attack') return 'attack';
if (key === 'decay') return 'decay';
if (key === 'useSound') return 'use sound';
if (key === 'emitSound') return 'emit sound';
return key;

238
client/src/main.ts
View File

@@ -14,6 +14,7 @@ import {
shouldProxyStreamUrl,
} from './audio/radioStationRuntime';
import { ItemEmitRuntime } from './audio/itemEmitRuntime';
import { PianoSynth, type PianoInstrumentId } from './audio/pianoSynth';
import { normalizeDegrees } from './audio/spatial';
import {
applyPastedText,
@@ -82,6 +83,29 @@ const RECONNECT_MAX_ATTEMPTS = 3;
const AUDIO_SUBSCRIPTION_REFRESH_MS = 500;
const TELEPORT_SQUARES_PER_SECOND = 20;
const TELEPORT_SYNC_INTERVAL_MS = 100;
const PIANO_WHITE_KEY_MIDI_BY_CODE: Record<string, number> = {
KeyA: 60,
KeyS: 62,
KeyD: 64,
KeyF: 65,
KeyG: 67,
KeyH: 69,
KeyJ: 71,
KeyK: 72,
KeyL: 74,
Semicolon: 76,
Quote: 77,
};
const PIANO_SHARP_KEY_MIDI_BY_CODE: Record<string, number> = {
KeyW: 61,
KeyE: 63,
KeyT: 66,
KeyY: 68,
KeyU: 70,
KeyO: 73,
KeyP: 75,
BracketRight: 78,
};
declare global {
interface Window {
@@ -224,6 +248,9 @@ let subscriptionRefreshPending = false;
let suppressItemPropertyEchoUntilMs = 0;
let activeTeleportLoopStop: (() => void) | null = null;
let activeTeleportLoopToken = 0;
let activePianoItemId: string | null = null;
const activePianoKeys = new Set<string>();
const activeRemotePianoKeys = new Set<string>();
let activeTeleport:
| {
startX: number;
@@ -238,6 +265,7 @@ let activeTeleport:
completionStatus: string;
}
| null = null;
const pianoSynth = new PianoSynth();
const signalingProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
const signalingUrl = `${signalingProtocol}://${window.location.host}/ws`;
@@ -759,6 +787,136 @@ function getItemSpatialConfig(item: WorldItem): { range: number; directional: bo
return { range, directional, facingDeg };
}
/** Resolves piano params with safe defaults for local play mode. */
function getPianoParams(item: WorldItem): { instrument: PianoInstrumentId; attack: number; decay: number; emitRange: number } {
const rawInstrument = String(item.params.instrument ?? 'piano').trim().toLowerCase();
const instrument: PianoInstrumentId =
rawInstrument === 'electric_piano' ||
rawInstrument === 'guitar' ||
rawInstrument === 'organ' ||
rawInstrument === 'bass' ||
rawInstrument === 'violin' ||
rawInstrument === 'synth_lead' ||
rawInstrument === 'drum_kit'
? rawInstrument
: 'piano';
const rawAttack = Number(item.params.attack);
const rawDecay = Number(item.params.decay);
const rawEmitRange = Number(item.params.emitRange ?? getItemTypeGlobalProperties(item.type).emitRange ?? 15);
return {
instrument,
attack: Math.max(0, Math.min(100, Number.isFinite(rawAttack) ? Math.round(rawAttack) : 15)),
decay: Math.max(0, Math.min(100, Number.isFinite(rawDecay) ? Math.round(rawDecay) : 45)),
emitRange: Math.max(5, Math.min(20, Number.isFinite(rawEmitRange) ? Math.round(rawEmitRange) : 15)),
};
}
/** Normalizes arbitrary instrument strings into supported piano synth ids. */
function normalizePianoInstrument(value: unknown): PianoInstrumentId {
const raw = String(value ?? 'piano').trim().toLowerCase();
if (raw === 'electric_piano') return 'electric_piano';
if (raw === 'guitar') return 'guitar';
if (raw === 'organ') return 'organ';
if (raw === 'bass') return 'bass';
if (raw === 'violin') return 'violin';
if (raw === 'synth_lead') return 'synth_lead';
if (raw === 'drum_kit') return 'drum_kit';
return 'piano';
}
/** Returns playable MIDI note for a piano-mode key code, or null when unmapped. */
function getPianoMidiForCode(code: string): number | null {
if (code in PIANO_WHITE_KEY_MIDI_BY_CODE) {
return PIANO_WHITE_KEY_MIDI_BY_CODE[code]!;
}
if (code in PIANO_SHARP_KEY_MIDI_BY_CODE) {
return PIANO_SHARP_KEY_MIDI_BY_CODE[code]!;
}
return null;
}
/** Starts local piano key mode for one used piano item. */
async function startPianoUseMode(itemId: string): Promise<void> {
const item = state.items.get(itemId);
if (!item || item.type !== 'piano') return;
activePianoItemId = itemId;
activePianoKeys.clear();
state.mode = 'pianoUse';
await audio.ensureContext();
updateStatus(`Piano mode: ${item.title}. Press Enter or Escape to stop.`);
audio.sfxUiBlip();
}
/** Exits local piano key mode and releases any held notes. */
function stopPianoUseMode(announce = true): void {
if (!activePianoItemId) return;
const itemId = activePianoItemId;
for (const code of Array.from(activePianoKeys)) {
const midi = getPianoMidiForCode(code);
if (midi === null) continue;
signaling.send({ type: 'item_piano_note', itemId, keyId: code, midi, on: false });
pianoSynth.noteOff(code);
}
activePianoItemId = null;
activePianoKeys.clear();
state.mode = 'normal';
if (announce) {
updateStatus('Stopped piano.');
audio.sfxUiCancel();
}
}
/** Plays one inbound piano note from another user using item spatial position. */
function playRemotePianoNote(note: {
itemId: string;
senderId: string;
keyId: string;
midi: number;
instrument: string;
attack: number;
decay: number;
x: number;
y: number;
emitRange: number;
}): void {
const ctx = audio.context;
const destination = audio.getOutputDestinationNode();
if (!ctx || !destination) return;
const runtimeKey = `${note.senderId}:${note.keyId}`;
if (activeRemotePianoKeys.has(runtimeKey)) return;
activeRemotePianoKeys.add(runtimeKey);
pianoSynth.noteOn(
runtimeKey,
Math.max(0, Math.min(127, Math.round(note.midi))),
normalizePianoInstrument(note.instrument),
Math.max(0, Math.min(100, Math.round(note.attack))),
Math.max(0, Math.min(100, Math.round(note.decay))),
{ audioCtx: ctx, destination },
{
x: note.x - state.player.x,
y: note.y - state.player.y,
range: Math.max(1, Math.round(note.emitRange)),
},
);
}
/** Stops one inbound piano note previously started for another user. */
function stopRemotePianoNote(senderId: string, keyId: string): void {
const runtimeKey = `${senderId}:${keyId}`;
if (!activeRemotePianoKeys.delete(runtimeKey)) return;
pianoSynth.noteOff(runtimeKey);
}
/** Stops all currently active remote piano notes for a sender id. */
function stopAllRemotePianoNotesForSender(senderId: string): void {
const prefix = `${senderId}:`;
for (const runtimeKey of Array.from(activeRemotePianoKeys)) {
if (!runtimeKey.startsWith(prefix)) continue;
activeRemotePianoKeys.delete(runtimeKey);
pianoSynth.noteOff(runtimeKey);
}
}
/** Enters help-view mode and announces the first help line. */
function openHelpViewer(): void {
if (helpViewerLines.length === 0) {
@@ -973,7 +1131,7 @@ function getItemPropertyValue(item: WorldItem, key: string): string {
function inferItemPropertyValueType(item: WorldItem, key: string): string | undefined {
if (key === 'useSound' || key === 'emitSound') return 'sound';
if (key === 'enabled' || key === 'use24Hour' || key === 'directional') return 'boolean';
if (key === 'mediaChannel' || key === 'mediaEffect' || key === 'emitEffect' || key === 'timeZone') return 'list';
if (key === 'mediaChannel' || key === 'mediaEffect' || key === 'emitEffect' || key === 'timeZone' || key === 'instrument') return 'list';
if (
key === 'x' ||
key === 'y' ||
@@ -986,6 +1144,8 @@ function inferItemPropertyValueType(item: WorldItem, key: string): string | unde
key === 'emitEffectValue' ||
key === 'facing' ||
key === 'emitRange' ||
key === 'attack' ||
key === 'decay' ||
key === 'sides' ||
key === 'number' ||
key === 'useCooldownMs'
@@ -1447,6 +1607,11 @@ function disconnect(): void {
lastSubscriptionRefreshTileY = Math.round(state.player.y);
stopTeleportLoopAudio();
activeTeleport = null;
stopPianoUseMode(false);
for (const key of Array.from(activeRemotePianoKeys)) {
activeRemotePianoKeys.delete(key);
pianoSynth.noteOff(key);
}
}
const onAppMessage = createOnMessageHandler({
@@ -1482,6 +1647,9 @@ const onAppMessage = createOnMessageHandler({
gain,
);
},
playRemotePianoNote,
stopRemotePianoNote,
stopAllRemotePianoNotesForSender,
TELEPORT_SOUND_URL,
TELEPORT_START_SOUND_URL,
getAudioLayers: () => audioLayers,
@@ -1543,6 +1711,20 @@ async function onSignalingMessage(message: IncomingMessage): Promise<void> {
startHeartbeat();
}
await onAppMessage(message);
if (
message.type === 'item_action_result' &&
message.ok &&
message.action === 'use' &&
typeof message.itemId === 'string'
) {
const item = state.items.get(message.itemId);
if (item?.type === 'piano') {
await startPianoUseMode(item.id);
}
}
if (activePianoItemId && !state.items.has(activePianoItemId)) {
stopPianoUseMode(false);
}
applyConfiguredPeerListenGains();
if (restartAnnouncement) {
setConnectionStatus(restartAnnouncement);
@@ -2015,6 +2197,44 @@ function handleMicGainEditModeInput(code: string, key: string, ctrlKey: boolean)
applyTextInputEdit(code, key, 8, ctrlKey, true);
}
/** Handles realtime keyboard performance while piano item mode is active. */
function handlePianoUseModeInput(code: string): void {
if (code === 'Escape' || code === 'Enter') {
stopPianoUseMode(true);
return;
}
const itemId = activePianoItemId;
if (!itemId) {
state.mode = 'normal';
return;
}
const item = state.items.get(itemId);
if (!item || item.type !== 'piano') {
stopPianoUseMode(false);
return;
}
const midi = getPianoMidiForCode(code);
if (midi === null) return;
if (activePianoKeys.has(code)) return;
activePianoKeys.add(code);
const ctx = audio.context;
const destination = audio.getOutputDestinationNode();
if (!ctx || !destination) return;
const config = getPianoParams(item);
const sourceX = item.carrierId === state.player.id ? state.player.x : item.x;
const sourceY = item.carrierId === state.player.id ? state.player.y : item.y;
pianoSynth.noteOn(
code,
midi,
config.instrument,
config.attack,
config.decay,
{ audioCtx: ctx, destination },
{ x: sourceX - state.player.x, y: sourceY - state.player.y, range: config.emitRange },
);
signaling.send({ type: 'item_piano_note', itemId, keyId: code, midi, on: true });
}
/** Handles effect menu list navigation and selection. */
function handleEffectSelectModeInput(code: string, key: string): void {
const control = handleListControlKey(code, key, EFFECT_SEQUENCE, state.effectSelectIndex, (effect) => effect.label);
@@ -2339,6 +2559,11 @@ function codeFromKey(key: string, location: number): string | null {
if (key === '/' || key === '?') return 'Slash';
if (key === ',' || key === '<') return 'Comma';
if (key === '.' || key === '>') return 'Period';
if (key === ';' || key === ':') return 'Semicolon';
if (key === "'" || key === '"') return 'Quote';
if (key === '[' || key === '{') return 'BracketLeft';
if (key === ']' || key === '}') return 'BracketRight';
if (key === '\\' || key === '|') return 'Backslash';
}
return null;
}
@@ -2411,6 +2636,7 @@ function setupInputHandlers(): void {
nickname: handleNicknameModeInput,
chat: handleChatModeInput,
micGainEdit: handleMicGainEditModeInput,
pianoUse: (currentCode) => handlePianoUseModeInput(currentCode),
effectSelect: (currentCode, currentKey) => handleEffectSelectModeInput(currentCode, currentKey),
helpView: (currentCode) => handleHelpViewModeInput(currentCode),
listUsers: (currentCode, currentKey) => handleListModeInput(currentCode, currentKey),
@@ -2431,6 +2657,16 @@ function setupInputHandlers(): void {
document.addEventListener('keyup', (event) => {
const code = normalizeInputCode(event);
if (state.mode === 'pianoUse' && code) {
if (activePianoKeys.delete(code)) {
pianoSynth.noteOff(code);
const itemId = activePianoItemId;
const midi = getPianoMidiForCode(code);
if (itemId && midi !== null) {
signaling.send({ type: 'item_piano_note', itemId, keyId: code, midi, on: false });
}
}
}
if (code) {
state.keysPressed[code] = false;
}

40
client/src/network/messageHandlers.ts
View File

@@ -46,9 +46,23 @@ type MessageHandlerDeps = {
sanitizeName: (value: string) => string;
randomFootstepUrl: () => string;
playRemoteSpatialStepOrTeleport: (url: string, peerX: number, peerY: number) => void;
playRemotePianoNote: (note: {
itemId: string;
senderId: string;
keyId: string;
midi: number;
instrument: string;
attack: number;
decay: number;
x: number;
y: number;
emitRange: number;
}) => void;
stopRemotePianoNote: (senderId: string, keyId: string) => void;
stopAllRemotePianoNotesForSender: (senderId: string) => void;
TELEPORT_SOUND_URL: string;
TELEPORT_START_SOUND_URL: string;
getAudioLayers: () => { world: boolean };
getAudioLayers: () => { world: boolean; item: boolean };
pushChatMessage: (message: string) => void;
classifySystemMessageSound: (message: string) => 'logon' | 'logout' | 'notify' | null;
SYSTEM_SOUND_URLS: { logon: string; logout: string; notify: string };
@@ -159,6 +173,7 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
if (peer) {
deps.updateStatus(`${peer.nickname} has left.`);
}
deps.stopAllRemotePianoNotesForSender(message.id);
deps.state.peers.delete(message.id);
deps.peerManager.removePeer(message.id);
break;
@@ -226,7 +241,7 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
if (message.action === 'use') {
deps.pushChatMessage(message.message);
const item = message.itemId ? deps.getItemById(message.itemId) : null;
if (!item?.useSound && item) {
if (!item?.useSound && item && item.type !== 'piano') {
deps.playLocateToneAt(item.x, item.y);
}
} else if (message.action !== 'update') {
@@ -248,6 +263,27 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
}
break;
}
case 'item_piano_note': {
if (!deps.getAudioLayers().item) break;
if (message.on) {
deps.playRemotePianoNote({
itemId: message.itemId,
senderId: message.senderId,
keyId: message.keyId,
midi: message.midi,
instrument: message.instrument,
attack: message.attack,
decay: message.decay,
x: message.x,
y: message.y,
emitRange: message.emitRange,
});
} else {
deps.stopRemotePianoNote(message.senderId, message.keyId);
}
break;
}
}
};
}

25
client/src/network/protocol.ts
View File

@@ -2,7 +2,7 @@ import { z } from 'zod';
export const itemSchema = z.object({
id: z.string(),
type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget']),
type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget', 'piano']),
title: z.string(),
x: z.number().int(),
y: z.number().int(),
@@ -42,10 +42,10 @@ export const welcomeMessageSchema = z.object({
.optional(),
uiDefinitions: z
.object({
itemTypeOrder: z.array(z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget'])),
itemTypeOrder: z.array(z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget', 'piano'])),
itemTypes: z.array(
z.object({
type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget']),
type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget', 'piano']),
label: z.string().optional(),
tooltip: z.string().optional(),
editableProperties: z.array(z.string()),
@@ -150,6 +150,21 @@ export const itemUseSoundSchema = z.object({
y: z.number().int(),
});
export const itemPianoNoteSchema = z.object({
type: z.literal('item_piano_note'),
itemId: z.string(),
senderId: z.string(),
keyId: z.string(),
midi: z.number().int().min(0).max(127),
on: z.boolean(),
instrument: z.string(),
attack: z.number().int().min(0).max(100),
decay: z.number().int().min(0).max(100),
x: z.number().int(),
y: z.number().int(),
emitRange: z.number().int().min(1),
});
export const incomingMessageSchema = z.discriminatedUnion('type', [
welcomeMessageSchema,
signalMessageSchema,
@@ -163,6 +178,7 @@ export const incomingMessageSchema = z.discriminatedUnion('type', [
itemRemoveSchema,
itemActionResultSchema,
itemUseSoundSchema,
itemPianoNoteSchema,
]);
export type IncomingMessage = z.infer<typeof incomingMessageSchema>;
@@ -173,11 +189,12 @@ export type OutgoingMessage =
| { type: 'update_nickname'; nickname: string }
| { type: 'chat_message'; message: string }
| { type: 'ping'; clientSentAt: number }
| { type: 'item_add'; itemType: 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget' }
| { type: 'item_add'; itemType: 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget' | 'piano' }
| { type: 'item_pickup'; itemId: string }
| { type: 'item_drop'; itemId: string; x: number; y: number }
| { type: 'item_delete'; itemId: string }
| { type: 'item_use'; itemId: string }
| { type: 'item_piano_note'; itemId: string; keyId: string; midi: number; on: boolean }
| {
type: 'item_update';
itemId: string;

4
client/src/render/canvasRenderer.ts
View File

@@ -89,6 +89,8 @@ export class CanvasRenderer {
? '#fbbf24'
: item.type === 'wheel'
? '#f97316'
: item.type === 'piano'
? '#c4b5fd'
: item.type === 'clock'
? '#86efac'
: item.type === 'widget'
@@ -103,6 +105,8 @@ export class CanvasRenderer {
? 'R'
: item.type === 'wheel'
? 'W'
: item.type === 'piano'
? 'P'
: item.type === 'clock'
? 'C'
: item.type === 'widget'

5
client/src/state/gameState.ts
View File

@@ -2,7 +2,7 @@ export const GRID_SIZE = 41;
export const HEARING_RADIUS = 20;
export const MOVE_COOLDOWN_MS = 200;
export type ItemType = 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget';
export type ItemType = 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget' | 'piano';
export type WorldItem = {
id: string;
@@ -36,7 +36,8 @@ export type GameMode =
| 'selectItem'
| 'itemProperties'
| 'itemPropertyEdit'
| 'itemPropertyOptionSelect';
| 'itemPropertyOptionSelect'
| 'pianoUse';
export type Player = {
id: string | null;