Add piano item type with realtime play mode and remote notes

This commit is contained in:
Jage9
2026-02-22 23:42:17 -05:00
parent 81c6af6399
commit 1319c044dd
23 changed files with 1014 additions and 23 deletions

View File

@@ -84,6 +84,10 @@
{ {
"keys": "Enter", "keys": "Enter",
"description": "Use item" "description": "Use item"
},
{
"keys": "Piano mode",
"description": "When using a piano: ASDFGHJKL;' plays C major notes, WETYUOP] plays sharps, Enter/Escape exits"
} }
] ]
}, },

View File

@@ -1,5 +1,5 @@
// Maintainer-controlled web client version. // Maintainer-controlled web client version.
// Format: YYYY.MM.DD Rn (example: 2026.02.20 R2) // Format: YYYY.MM.DD Rn (example: 2026.02.20 R2)
window.CHGRID_WEB_VERSION = "2026.02.22 R198"; window.CHGRID_WEB_VERSION = "2026.02.22 R199";
// Optional display timezone for timestamps. Falls back to America/Detroit if unset/invalid. // Optional display timezone for timestamps. Falls back to America/Detroit if unset/invalid.
window.CHGRID_TIME_ZONE = "America/Detroit"; window.CHGRID_TIME_ZONE = "America/Detroit";

View File

@@ -0,0 +1,330 @@
import { resolveSpatialMix } from './spatial';
export const PIANO_INSTRUMENT_OPTIONS = [
'piano',
'electric_piano',
'guitar',
'organ',
'bass',
'violin',
'synth_lead',
'drum_kit',
] as const;
export type PianoInstrumentId = (typeof PIANO_INSTRUMENT_OPTIONS)[number];
type VoiceRuntime = {
gain: GainNode;
panner: StereoPannerNode | null;
oscillators: OscillatorNode[];
releaseSeconds: number;
};
type PianoContext = {
audioCtx: AudioContext;
destination: AudioNode;
};
type PianoSpatialSource = {
x: number;
y: number;
range: number;
};
type InstrumentPreset = {
oscillators: Array<{ type: OscillatorType; detune?: number; gain?: number; ratio?: number }>;
filter?: { type: BiquadFilterType; frequency: number; q?: number };
gain: number;
releaseScale?: number;
};
const PRESETS: Record<Exclude<PianoInstrumentId, 'drum_kit'>, InstrumentPreset> = {
piano: {
oscillators: [
{ type: 'triangle', gain: 1 },
{ type: 'sine', ratio: 2, gain: 0.28 },
],
filter: { type: 'lowpass', frequency: 5200, q: 0.7 },
gain: 0.32,
releaseScale: 0.9,
},
electric_piano: {
oscillators: [
{ type: 'sine', gain: 1 },
{ type: 'triangle', detune: 5, gain: 0.35 },
],
filter: { type: 'lowpass', frequency: 4200, q: 0.8 },
gain: 0.3,
releaseScale: 0.8,
},
guitar: {
oscillators: [
{ type: 'triangle', gain: 1 },
{ type: 'sawtooth', detune: -3, gain: 0.2 },
],
filter: { type: 'lowpass', frequency: 3200, q: 0.9 },
gain: 0.24,
releaseScale: 0.7,
},
organ: {
oscillators: [
{ type: 'square', gain: 0.8 },
{ type: 'sine', ratio: 2, gain: 0.28 },
{ type: 'sine', ratio: 3, gain: 0.2 },
],
filter: { type: 'lowpass', frequency: 6500, q: 0.6 },
gain: 0.18,
releaseScale: 1.4,
},
bass: {
oscillators: [
{ type: 'sawtooth', gain: 0.9 },
{ type: 'square', ratio: 0.5, gain: 0.25 },
],
filter: { type: 'lowpass', frequency: 1500, q: 1.1 },
gain: 0.28,
releaseScale: 0.9,
},
violin: {
oscillators: [
{ type: 'sawtooth', gain: 0.8 },
{ type: 'triangle', detune: 3, gain: 0.35 },
],
filter: { type: 'lowpass', frequency: 3600, q: 1.0 },
gain: 0.24,
releaseScale: 1.5,
},
synth_lead: {
oscillators: [
{ type: 'sawtooth', gain: 0.85 },
{ type: 'square', detune: 6, gain: 0.3 },
],
filter: { type: 'lowpass', frequency: 5400, q: 0.9 },
gain: 0.2,
releaseScale: 1,
},
};
/** Maps 0..100 control values to note attack seconds. */
function attackPercentToSeconds(value: number): number {
const clamped = Math.max(0, Math.min(100, value));
return 0.002 + (clamped / 100) * 0.6;
}
/** Maps 0..100 control values to note decay/release seconds. */
function decayPercentToSeconds(value: number): number {
const clamped = Math.max(0, Math.min(100, value));
return 0.05 + (clamped / 100) * 2.7;
}
/** Converts midi note number to frequency in hertz. */
function midiToFrequency(midi: number): number {
return 440 * Math.pow(2, (midi - 69) / 12);
}
/** Small helper to safely stop audio nodes. */
function safeStop(oscillator: OscillatorNode, when: number): void {
try {
oscillator.stop(when);
} catch {
// Ignore already-stopped oscillators.
}
}
export class PianoSynth {
private readonly voices = new Map<string, VoiceRuntime>();
private readonly drumNoiseBuffers = new WeakMap<AudioContext, AudioBuffer>();
/** Stops and disconnects all active notes. */
stopAll(): void {
for (const key of Array.from(this.voices.keys())) {
this.noteOff(key);
}
}
/** Starts one note for a specific keyboard key id. */
noteOn(
keyId: string,
midi: number,
instrument: PianoInstrumentId,
attackPercent: number,
decayPercent: number,
context: PianoContext,
spatial: PianoSpatialSource,
): void {
if (this.voices.has(keyId)) return;
if (instrument === 'drum_kit') {
this.playDrumHit(keyId, context, spatial, attackPercent, decayPercent);
return;
}
const preset = PRESETS[instrument] ?? PRESETS.piano;
const now = context.audioCtx.currentTime;
const attackSeconds = attackPercentToSeconds(attackPercent);
const decaySeconds = decayPercentToSeconds(decayPercent);
const releaseSeconds = Math.max(0.02, decaySeconds * (preset.releaseScale ?? 1));
const spatialMix = resolveSpatialMix({
dx: spatial.x,
dy: spatial.y,
range: spatial.range,
baseGain: 1,
});
if (!spatialMix || spatialMix.gain <= 0) return;
const voiceGain = context.audioCtx.createGain();
voiceGain.gain.setValueAtTime(0.0001, now);
const peakGain = Math.max(0.0001, preset.gain * spatialMix.gain);
const sustainGain = Math.max(0.0001, peakGain * 0.55);
voiceGain.gain.exponentialRampToValueAtTime(peakGain, now + attackSeconds);
voiceGain.gain.exponentialRampToValueAtTime(sustainGain, now + attackSeconds + decaySeconds * 0.6);
let tailNode: AudioNode = voiceGain;
if (preset.filter) {
const filter = context.audioCtx.createBiquadFilter();
filter.type = preset.filter.type;
filter.frequency.setValueAtTime(preset.filter.frequency, now);
filter.Q.setValueAtTime(preset.filter.q ?? 0.7, now);
voiceGain.connect(filter);
tailNode = filter;
}
let panner: StereoPannerNode | null = null;
if (typeof context.audioCtx.createStereoPanner === 'function') {
panner = context.audioCtx.createStereoPanner();
panner.pan.setValueAtTime(spatialMix.pan, now);
tailNode.connect(panner).connect(context.destination);
} else {
tailNode.connect(context.destination);
}
const frequency = midiToFrequency(midi);
const oscillators: OscillatorNode[] = [];
for (const partial of preset.oscillators) {
const oscillator = context.audioCtx.createOscillator();
oscillator.type = partial.type;
oscillator.frequency.setValueAtTime(frequency * (partial.ratio ?? 1), now);
oscillator.detune.setValueAtTime(partial.detune ?? 0, now);
const oscGain = context.audioCtx.createGain();
oscGain.gain.setValueAtTime(partial.gain ?? 1, now);
oscillator.connect(oscGain).connect(voiceGain);
oscillator.start(now);
oscillators.push(oscillator);
}
this.voices.set(keyId, {
gain: voiceGain,
panner,
oscillators,
releaseSeconds,
});
}
/** Releases one active note tied to a keyboard key id. */
noteOff(keyId: string): void {
const voice = this.voices.get(keyId);
if (!voice) return;
this.voices.delete(keyId);
const now = voice.gain.context.currentTime;
const currentGain = Math.max(0.0001, voice.gain.gain.value);
voice.gain.gain.cancelScheduledValues(now);
voice.gain.gain.setValueAtTime(currentGain, now);
voice.gain.gain.exponentialRampToValueAtTime(0.0001, now + voice.releaseSeconds);
for (const oscillator of voice.oscillators) {
safeStop(oscillator, now + voice.releaseSeconds + 0.02);
}
window.setTimeout(() => {
try {
voice.gain.disconnect();
} catch {
// Ignore stale disconnects.
}
if (voice.panner) {
try {
voice.panner.disconnect();
} catch {
// Ignore stale disconnects.
}
}
}, Math.max(60, Math.round((voice.releaseSeconds + 0.04) * 1000)));
}
/** Plays one synthesized drum hit for drum-kit instrument mode. */
private playDrumHit(
keyId: string,
context: PianoContext,
spatial: PianoSpatialSource,
attackPercent: number,
decayPercent: number,
): void {
const now = context.audioCtx.currentTime;
const spatialMix = resolveSpatialMix({
dx: spatial.x,
dy: spatial.y,
range: spatial.range,
baseGain: 1,
});
if (!spatialMix || spatialMix.gain <= 0) return;
const typeIndex = Math.abs(this.hashKey(keyId)) % 4;
const decaySeconds = 0.03 + decayPercentToSeconds(decayPercent) * 0.45;
const attackSeconds = Math.max(0.001, attackPercentToSeconds(attackPercent) * 0.2);
const gain = context.audioCtx.createGain();
gain.gain.setValueAtTime(0.0001, now);
gain.gain.exponentialRampToValueAtTime(0.22 * spatialMix.gain, now + attackSeconds);
gain.gain.exponentialRampToValueAtTime(0.0001, now + decaySeconds);
let tailNode: AudioNode = gain;
if (typeof context.audioCtx.createStereoPanner === 'function') {
const panner = context.audioCtx.createStereoPanner();
panner.pan.setValueAtTime(spatialMix.pan, now);
tailNode.connect(panner).connect(context.destination);
tailNode = panner;
} else {
tailNode.connect(context.destination);
}
if (typeIndex === 0) {
const kick = context.audioCtx.createOscillator();
kick.type = 'sine';
kick.frequency.setValueAtTime(150, now);
kick.frequency.exponentialRampToValueAtTime(45, now + decaySeconds * 0.85);
kick.connect(gain);
kick.start(now);
safeStop(kick, now + decaySeconds + 0.04);
return;
}
const noise = context.audioCtx.createBufferSource();
noise.buffer = this.getNoiseBuffer(context.audioCtx);
const noiseFilter = context.audioCtx.createBiquadFilter();
noiseFilter.type = typeIndex === 1 ? 'highpass' : typeIndex === 2 ? 'bandpass' : 'lowpass';
noiseFilter.frequency.setValueAtTime(typeIndex === 1 ? 1700 : typeIndex === 2 ? 900 : 1300, now);
noise.connect(noiseFilter).connect(gain);
noise.start(now);
safeStop(noise, now + decaySeconds + 0.03);
}
/** Returns deterministic hash for key ids to map drum voice variants. */
private hashKey(value: string): number {
let out = 0;
for (let index = 0; index < value.length; index += 1) {
out = ((out << 5) - out + value.charCodeAt(index)) | 0;
}
return out;
}
/** Returns or lazily builds short white-noise buffer for percussion synthesis. */
private getNoiseBuffer(audioCtx: AudioContext): AudioBuffer {
const existing = this.drumNoiseBuffers.get(audioCtx);
if (existing) return existing;
const length = Math.max(1, Math.floor(audioCtx.sampleRate * 0.5));
const buffer = audioCtx.createBuffer(1, length, audioCtx.sampleRate);
const data = buffer.getChannelData(0);
for (let index = 0; index < length; index += 1) {
data[index] = Math.random() * 2 - 1;
}
this.drumNoiseBuffers.set(audioCtx, buffer);
return buffer;
}
}

View File

@@ -309,6 +309,8 @@ export function createItemPropertyEditor(deps: EditorDeps): {
propertyKey === 'mediaVolume' || propertyKey === 'mediaVolume' ||
propertyKey === 'emitVolume' || propertyKey === 'emitVolume' ||
propertyKey === 'emitRange' || propertyKey === 'emitRange' ||
propertyKey === 'attack' ||
propertyKey === 'decay' ||
propertyKey === 'sides' || propertyKey === 'sides' ||
propertyKey === 'number' propertyKey === 'number'
) { ) {

View File

@@ -45,11 +45,22 @@ const DEFAULT_CLOCK_TIME_ZONE_OPTIONS = [
'UTC', 'UTC',
] as const; ] as const;
const DEFAULT_ITEM_TYPE_SEQUENCE: ItemType[] = ['clock', 'dice', 'radio_station', 'wheel', 'widget']; const DEFAULT_ITEM_TYPE_SEQUENCE: ItemType[] = ['clock', 'dice', 'piano', 'radio_station', 'wheel', 'widget'];
const DEFAULT_PIANO_INSTRUMENT_OPTIONS = [
'piano',
'electric_piano',
'guitar',
'organ',
'bass',
'violin',
'synth_lead',
'drum_kit',
] as const;
const DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES: Record<ItemType, string[]> = { const DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES: Record<ItemType, string[]> = {
radio_station: ['title', 'streamUrl', 'enabled', 'mediaVolume', 'mediaChannel', 'mediaEffect', 'mediaEffectValue', 'facing', 'emitRange'], radio_station: ['title', 'streamUrl', 'enabled', 'mediaVolume', 'mediaChannel', 'mediaEffect', 'mediaEffectValue', 'facing', 'emitRange'],
dice: ['title', 'sides', 'number'], dice: ['title', 'sides', 'number'],
piano: ['title', 'instrument', 'attack', 'decay', 'emitRange'],
wheel: ['title', 'spaces'], wheel: ['title', 'spaces'],
clock: ['title', 'timeZone', 'use24Hour'], clock: ['title', 'timeZone', 'use24Hour'],
widget: ['title', 'enabled', 'directional', 'facing', 'emitRange', 'emitVolume', 'emitSoundSpeed', 'emitSoundTempo', 'emitEffect', 'emitEffectValue', 'useSound', 'emitSound'], widget: ['title', 'enabled', 'directional', 'facing', 'emitRange', 'emitVolume', 'emitSoundSpeed', 'emitSoundTempo', 'emitEffect', 'emitEffectValue', 'useSound', 'emitSound'],
@@ -58,6 +69,7 @@ const DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES: Record<ItemType, string[]> = {
const DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES: Record<ItemType, Record<string, string | number | boolean>> = { const DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES: Record<ItemType, Record<string, string | number | boolean>> = {
radio_station: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 20, directional: true, emitSoundSpeed: 50, emitSoundTempo: 50 }, radio_station: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 20, directional: true, emitSoundSpeed: 50, emitSoundTempo: 50 },
dice: { useSound: 'sounds/roll.ogg', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 }, dice: { useSound: 'sounds/roll.ogg', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
piano: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
wheel: { useSound: 'sounds/spin.ogg', emitSound: 'none', useCooldownMs: 4000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 }, wheel: { useSound: 'sounds/spin.ogg', emitSound: 'none', useCooldownMs: 4000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
clock: { useSound: 'none', emitSound: 'sounds/clock.ogg', useCooldownMs: 1000, emitRange: 10, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 }, clock: { useSound: 'none', emitSound: 'sounds/clock.ogg', useCooldownMs: 1000, emitRange: 10, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
widget: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 }, widget: { useSound: 'none', emitSound: 'none', useCooldownMs: 1000, emitRange: 15, directional: false, emitSoundSpeed: 50, emitSoundTempo: 50 },
@@ -93,6 +105,7 @@ let itemTypeSequence: ItemType[] = [...DEFAULT_ITEM_TYPE_SEQUENCE];
let itemTypeLabels: Record<ItemType, string> = { let itemTypeLabels: Record<ItemType, string> = {
radio_station: 'radio', radio_station: 'radio',
dice: 'dice', dice: 'dice',
piano: 'piano',
wheel: 'wheel', wheel: 'wheel',
clock: 'clock', clock: 'clock',
widget: 'widget', widget: 'widget',
@@ -101,6 +114,7 @@ let itemTypeTooltips: Partial<Record<ItemType, string>> = {};
let itemTypeEditableProperties: Record<ItemType, string[]> = { let itemTypeEditableProperties: Record<ItemType, string[]> = {
radio_station: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.radio_station], radio_station: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.radio_station],
dice: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.dice], dice: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.dice],
piano: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.piano],
wheel: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.wheel], wheel: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.wheel],
clock: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.clock], clock: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.clock],
widget: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.widget], widget: [...DEFAULT_ITEM_TYPE_EDITABLE_PROPERTIES.widget],
@@ -108,6 +122,7 @@ let itemTypeEditableProperties: Record<ItemType, string[]> = {
let itemTypeGlobalProperties: Record<ItemType, Record<string, string | number | boolean>> = { let itemTypeGlobalProperties: Record<ItemType, Record<string, string | number | boolean>> = {
radio_station: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.radio_station }, radio_station: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.radio_station },
dice: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.dice }, dice: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.dice },
piano: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.piano },
wheel: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.wheel }, wheel: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.wheel },
clock: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.clock }, clock: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.clock },
widget: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.widget }, widget: { ...DEFAULT_ITEM_TYPE_GLOBAL_PROPERTIES.widget },
@@ -116,6 +131,7 @@ let optionItemPropertyValues: Partial<Record<string, string[]>> = {
mediaEffect: EFFECT_SEQUENCE.map((effect) => effect.id), mediaEffect: EFFECT_SEQUENCE.map((effect) => effect.id),
emitEffect: EFFECT_SEQUENCE.map((effect) => effect.id), emitEffect: EFFECT_SEQUENCE.map((effect) => effect.id),
mediaChannel: [...RADIO_CHANNEL_OPTIONS], mediaChannel: [...RADIO_CHANNEL_OPTIONS],
instrument: [...DEFAULT_PIANO_INSTRUMENT_OPTIONS],
timeZone: [...DEFAULT_CLOCK_TIME_ZONE_OPTIONS], timeZone: [...DEFAULT_CLOCK_TIME_ZONE_OPTIONS],
}; };
let itemTypePropertyMetadata: Partial<Record<ItemType, Record<string, ItemPropertyMetadata>>> = {}; let itemTypePropertyMetadata: Partial<Record<ItemType, Record<string, ItemPropertyMetadata>>> = {};
@@ -221,6 +237,9 @@ export function itemPropertyLabel(key: string): string {
if (key === 'mediaEffectValue') return 'media effect value'; if (key === 'mediaEffectValue') return 'media effect value';
if (key === 'emitEffect') return 'emit effect'; if (key === 'emitEffect') return 'emit effect';
if (key === 'emitEffectValue') return 'emit effect value'; if (key === 'emitEffectValue') return 'emit effect value';
if (key === 'instrument') return 'instrument';
if (key === 'attack') return 'attack';
if (key === 'decay') return 'decay';
if (key === 'useSound') return 'use sound'; if (key === 'useSound') return 'use sound';
if (key === 'emitSound') return 'emit sound'; if (key === 'emitSound') return 'emit sound';
return key; return key;

View File

@@ -14,6 +14,7 @@ import {
shouldProxyStreamUrl, shouldProxyStreamUrl,
} from './audio/radioStationRuntime'; } from './audio/radioStationRuntime';
import { ItemEmitRuntime } from './audio/itemEmitRuntime'; import { ItemEmitRuntime } from './audio/itemEmitRuntime';
import { PianoSynth, type PianoInstrumentId } from './audio/pianoSynth';
import { normalizeDegrees } from './audio/spatial'; import { normalizeDegrees } from './audio/spatial';
import { import {
applyPastedText, applyPastedText,
@@ -82,6 +83,29 @@ const RECONNECT_MAX_ATTEMPTS = 3;
const AUDIO_SUBSCRIPTION_REFRESH_MS = 500; const AUDIO_SUBSCRIPTION_REFRESH_MS = 500;
const TELEPORT_SQUARES_PER_SECOND = 20; const TELEPORT_SQUARES_PER_SECOND = 20;
const TELEPORT_SYNC_INTERVAL_MS = 100; const TELEPORT_SYNC_INTERVAL_MS = 100;
const PIANO_WHITE_KEY_MIDI_BY_CODE: Record<string, number> = {
KeyA: 60,
KeyS: 62,
KeyD: 64,
KeyF: 65,
KeyG: 67,
KeyH: 69,
KeyJ: 71,
KeyK: 72,
KeyL: 74,
Semicolon: 76,
Quote: 77,
};
const PIANO_SHARP_KEY_MIDI_BY_CODE: Record<string, number> = {
KeyW: 61,
KeyE: 63,
KeyT: 66,
KeyY: 68,
KeyU: 70,
KeyO: 73,
KeyP: 75,
BracketRight: 78,
};
declare global { declare global {
interface Window { interface Window {
@@ -224,6 +248,9 @@ let subscriptionRefreshPending = false;
let suppressItemPropertyEchoUntilMs = 0; let suppressItemPropertyEchoUntilMs = 0;
let activeTeleportLoopStop: (() => void) | null = null; let activeTeleportLoopStop: (() => void) | null = null;
let activeTeleportLoopToken = 0; let activeTeleportLoopToken = 0;
let activePianoItemId: string | null = null;
const activePianoKeys = new Set<string>();
const activeRemotePianoKeys = new Set<string>();
let activeTeleport: let activeTeleport:
| { | {
startX: number; startX: number;
@@ -238,6 +265,7 @@ let activeTeleport:
completionStatus: string; completionStatus: string;
} }
| null = null; | null = null;
const pianoSynth = new PianoSynth();
const signalingProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws'; const signalingProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
const signalingUrl = `${signalingProtocol}://${window.location.host}/ws`; const signalingUrl = `${signalingProtocol}://${window.location.host}/ws`;
@@ -759,6 +787,136 @@ function getItemSpatialConfig(item: WorldItem): { range: number; directional: bo
return { range, directional, facingDeg }; return { range, directional, facingDeg };
} }
/** Resolves piano params with safe defaults for local play mode. */
function getPianoParams(item: WorldItem): { instrument: PianoInstrumentId; attack: number; decay: number; emitRange: number } {
const rawInstrument = String(item.params.instrument ?? 'piano').trim().toLowerCase();
const instrument: PianoInstrumentId =
rawInstrument === 'electric_piano' ||
rawInstrument === 'guitar' ||
rawInstrument === 'organ' ||
rawInstrument === 'bass' ||
rawInstrument === 'violin' ||
rawInstrument === 'synth_lead' ||
rawInstrument === 'drum_kit'
? rawInstrument
: 'piano';
const rawAttack = Number(item.params.attack);
const rawDecay = Number(item.params.decay);
const rawEmitRange = Number(item.params.emitRange ?? getItemTypeGlobalProperties(item.type).emitRange ?? 15);
return {
instrument,
attack: Math.max(0, Math.min(100, Number.isFinite(rawAttack) ? Math.round(rawAttack) : 15)),
decay: Math.max(0, Math.min(100, Number.isFinite(rawDecay) ? Math.round(rawDecay) : 45)),
emitRange: Math.max(5, Math.min(20, Number.isFinite(rawEmitRange) ? Math.round(rawEmitRange) : 15)),
};
}
/** Normalizes arbitrary instrument strings into supported piano synth ids. */
function normalizePianoInstrument(value: unknown): PianoInstrumentId {
const raw = String(value ?? 'piano').trim().toLowerCase();
if (raw === 'electric_piano') return 'electric_piano';
if (raw === 'guitar') return 'guitar';
if (raw === 'organ') return 'organ';
if (raw === 'bass') return 'bass';
if (raw === 'violin') return 'violin';
if (raw === 'synth_lead') return 'synth_lead';
if (raw === 'drum_kit') return 'drum_kit';
return 'piano';
}
/** Returns playable MIDI note for a piano-mode key code, or null when unmapped. */
function getPianoMidiForCode(code: string): number | null {
if (code in PIANO_WHITE_KEY_MIDI_BY_CODE) {
return PIANO_WHITE_KEY_MIDI_BY_CODE[code]!;
}
if (code in PIANO_SHARP_KEY_MIDI_BY_CODE) {
return PIANO_SHARP_KEY_MIDI_BY_CODE[code]!;
}
return null;
}
/** Starts local piano key mode for one used piano item. */
async function startPianoUseMode(itemId: string): Promise<void> {
const item = state.items.get(itemId);
if (!item || item.type !== 'piano') return;
activePianoItemId = itemId;
activePianoKeys.clear();
state.mode = 'pianoUse';
await audio.ensureContext();
updateStatus(`Piano mode: ${item.title}. Press Enter or Escape to stop.`);
audio.sfxUiBlip();
}
/** Exits local piano key mode and releases any held notes. */
function stopPianoUseMode(announce = true): void {
if (!activePianoItemId) return;
const itemId = activePianoItemId;
for (const code of Array.from(activePianoKeys)) {
const midi = getPianoMidiForCode(code);
if (midi === null) continue;
signaling.send({ type: 'item_piano_note', itemId, keyId: code, midi, on: false });
pianoSynth.noteOff(code);
}
activePianoItemId = null;
activePianoKeys.clear();
state.mode = 'normal';
if (announce) {
updateStatus('Stopped piano.');
audio.sfxUiCancel();
}
}
/** Plays one inbound piano note from another user using item spatial position. */
function playRemotePianoNote(note: {
itemId: string;
senderId: string;
keyId: string;
midi: number;
instrument: string;
attack: number;
decay: number;
x: number;
y: number;
emitRange: number;
}): void {
const ctx = audio.context;
const destination = audio.getOutputDestinationNode();
if (!ctx || !destination) return;
const runtimeKey = `${note.senderId}:${note.keyId}`;
if (activeRemotePianoKeys.has(runtimeKey)) return;
activeRemotePianoKeys.add(runtimeKey);
pianoSynth.noteOn(
runtimeKey,
Math.max(0, Math.min(127, Math.round(note.midi))),
normalizePianoInstrument(note.instrument),
Math.max(0, Math.min(100, Math.round(note.attack))),
Math.max(0, Math.min(100, Math.round(note.decay))),
{ audioCtx: ctx, destination },
{
x: note.x - state.player.x,
y: note.y - state.player.y,
range: Math.max(1, Math.round(note.emitRange)),
},
);
}
/** Stops one inbound piano note previously started for another user. */
function stopRemotePianoNote(senderId: string, keyId: string): void {
const runtimeKey = `${senderId}:${keyId}`;
if (!activeRemotePianoKeys.delete(runtimeKey)) return;
pianoSynth.noteOff(runtimeKey);
}
/** Stops all currently active remote piano notes for a sender id. */
function stopAllRemotePianoNotesForSender(senderId: string): void {
const prefix = `${senderId}:`;
for (const runtimeKey of Array.from(activeRemotePianoKeys)) {
if (!runtimeKey.startsWith(prefix)) continue;
activeRemotePianoKeys.delete(runtimeKey);
pianoSynth.noteOff(runtimeKey);
}
}
/** Enters help-view mode and announces the first help line. */ /** Enters help-view mode and announces the first help line. */
function openHelpViewer(): void { function openHelpViewer(): void {
if (helpViewerLines.length === 0) { if (helpViewerLines.length === 0) {
@@ -973,7 +1131,7 @@ function getItemPropertyValue(item: WorldItem, key: string): string {
function inferItemPropertyValueType(item: WorldItem, key: string): string | undefined { function inferItemPropertyValueType(item: WorldItem, key: string): string | undefined {
if (key === 'useSound' || key === 'emitSound') return 'sound'; if (key === 'useSound' || key === 'emitSound') return 'sound';
if (key === 'enabled' || key === 'use24Hour' || key === 'directional') return 'boolean'; if (key === 'enabled' || key === 'use24Hour' || key === 'directional') return 'boolean';
if (key === 'mediaChannel' || key === 'mediaEffect' || key === 'emitEffect' || key === 'timeZone') return 'list'; if (key === 'mediaChannel' || key === 'mediaEffect' || key === 'emitEffect' || key === 'timeZone' || key === 'instrument') return 'list';
if ( if (
key === 'x' || key === 'x' ||
key === 'y' || key === 'y' ||
@@ -986,6 +1144,8 @@ function inferItemPropertyValueType(item: WorldItem, key: string): string | unde
key === 'emitEffectValue' || key === 'emitEffectValue' ||
key === 'facing' || key === 'facing' ||
key === 'emitRange' || key === 'emitRange' ||
key === 'attack' ||
key === 'decay' ||
key === 'sides' || key === 'sides' ||
key === 'number' || key === 'number' ||
key === 'useCooldownMs' key === 'useCooldownMs'
@@ -1447,6 +1607,11 @@ function disconnect(): void {
lastSubscriptionRefreshTileY = Math.round(state.player.y); lastSubscriptionRefreshTileY = Math.round(state.player.y);
stopTeleportLoopAudio(); stopTeleportLoopAudio();
activeTeleport = null; activeTeleport = null;
stopPianoUseMode(false);
for (const key of Array.from(activeRemotePianoKeys)) {
activeRemotePianoKeys.delete(key);
pianoSynth.noteOff(key);
}
} }
const onAppMessage = createOnMessageHandler({ const onAppMessage = createOnMessageHandler({
@@ -1482,6 +1647,9 @@ const onAppMessage = createOnMessageHandler({
gain, gain,
); );
}, },
playRemotePianoNote,
stopRemotePianoNote,
stopAllRemotePianoNotesForSender,
TELEPORT_SOUND_URL, TELEPORT_SOUND_URL,
TELEPORT_START_SOUND_URL, TELEPORT_START_SOUND_URL,
getAudioLayers: () => audioLayers, getAudioLayers: () => audioLayers,
@@ -1543,6 +1711,20 @@ async function onSignalingMessage(message: IncomingMessage): Promise<void> {
startHeartbeat(); startHeartbeat();
} }
await onAppMessage(message); await onAppMessage(message);
if (
message.type === 'item_action_result' &&
message.ok &&
message.action === 'use' &&
typeof message.itemId === 'string'
) {
const item = state.items.get(message.itemId);
if (item?.type === 'piano') {
await startPianoUseMode(item.id);
}
}
if (activePianoItemId && !state.items.has(activePianoItemId)) {
stopPianoUseMode(false);
}
applyConfiguredPeerListenGains(); applyConfiguredPeerListenGains();
if (restartAnnouncement) { if (restartAnnouncement) {
setConnectionStatus(restartAnnouncement); setConnectionStatus(restartAnnouncement);
@@ -2015,6 +2197,44 @@ function handleMicGainEditModeInput(code: string, key: string, ctrlKey: boolean)
applyTextInputEdit(code, key, 8, ctrlKey, true); applyTextInputEdit(code, key, 8, ctrlKey, true);
} }
/** Handles realtime keyboard performance while piano item mode is active. */
function handlePianoUseModeInput(code: string): void {
if (code === 'Escape' || code === 'Enter') {
stopPianoUseMode(true);
return;
}
const itemId = activePianoItemId;
if (!itemId) {
state.mode = 'normal';
return;
}
const item = state.items.get(itemId);
if (!item || item.type !== 'piano') {
stopPianoUseMode(false);
return;
}
const midi = getPianoMidiForCode(code);
if (midi === null) return;
if (activePianoKeys.has(code)) return;
activePianoKeys.add(code);
const ctx = audio.context;
const destination = audio.getOutputDestinationNode();
if (!ctx || !destination) return;
const config = getPianoParams(item);
const sourceX = item.carrierId === state.player.id ? state.player.x : item.x;
const sourceY = item.carrierId === state.player.id ? state.player.y : item.y;
pianoSynth.noteOn(
code,
midi,
config.instrument,
config.attack,
config.decay,
{ audioCtx: ctx, destination },
{ x: sourceX - state.player.x, y: sourceY - state.player.y, range: config.emitRange },
);
signaling.send({ type: 'item_piano_note', itemId, keyId: code, midi, on: true });
}
/** Handles effect menu list navigation and selection. */ /** Handles effect menu list navigation and selection. */
function handleEffectSelectModeInput(code: string, key: string): void { function handleEffectSelectModeInput(code: string, key: string): void {
const control = handleListControlKey(code, key, EFFECT_SEQUENCE, state.effectSelectIndex, (effect) => effect.label); const control = handleListControlKey(code, key, EFFECT_SEQUENCE, state.effectSelectIndex, (effect) => effect.label);
@@ -2339,6 +2559,11 @@ function codeFromKey(key: string, location: number): string | null {
if (key === '/' || key === '?') return 'Slash'; if (key === '/' || key === '?') return 'Slash';
if (key === ',' || key === '<') return 'Comma'; if (key === ',' || key === '<') return 'Comma';
if (key === '.' || key === '>') return 'Period'; if (key === '.' || key === '>') return 'Period';
if (key === ';' || key === ':') return 'Semicolon';
if (key === "'" || key === '"') return 'Quote';
if (key === '[' || key === '{') return 'BracketLeft';
if (key === ']' || key === '}') return 'BracketRight';
if (key === '\\' || key === '|') return 'Backslash';
} }
return null; return null;
} }
@@ -2411,6 +2636,7 @@ function setupInputHandlers(): void {
nickname: handleNicknameModeInput, nickname: handleNicknameModeInput,
chat: handleChatModeInput, chat: handleChatModeInput,
micGainEdit: handleMicGainEditModeInput, micGainEdit: handleMicGainEditModeInput,
pianoUse: (currentCode) => handlePianoUseModeInput(currentCode),
effectSelect: (currentCode, currentKey) => handleEffectSelectModeInput(currentCode, currentKey), effectSelect: (currentCode, currentKey) => handleEffectSelectModeInput(currentCode, currentKey),
helpView: (currentCode) => handleHelpViewModeInput(currentCode), helpView: (currentCode) => handleHelpViewModeInput(currentCode),
listUsers: (currentCode, currentKey) => handleListModeInput(currentCode, currentKey), listUsers: (currentCode, currentKey) => handleListModeInput(currentCode, currentKey),
@@ -2431,6 +2657,16 @@ function setupInputHandlers(): void {
document.addEventListener('keyup', (event) => { document.addEventListener('keyup', (event) => {
const code = normalizeInputCode(event); const code = normalizeInputCode(event);
if (state.mode === 'pianoUse' && code) {
if (activePianoKeys.delete(code)) {
pianoSynth.noteOff(code);
const itemId = activePianoItemId;
const midi = getPianoMidiForCode(code);
if (itemId && midi !== null) {
signaling.send({ type: 'item_piano_note', itemId, keyId: code, midi, on: false });
}
}
}
if (code) { if (code) {
state.keysPressed[code] = false; state.keysPressed[code] = false;
} }

View File

@@ -46,9 +46,23 @@ type MessageHandlerDeps = {
sanitizeName: (value: string) => string; sanitizeName: (value: string) => string;
randomFootstepUrl: () => string; randomFootstepUrl: () => string;
playRemoteSpatialStepOrTeleport: (url: string, peerX: number, peerY: number) => void; playRemoteSpatialStepOrTeleport: (url: string, peerX: number, peerY: number) => void;
playRemotePianoNote: (note: {
itemId: string;
senderId: string;
keyId: string;
midi: number;
instrument: string;
attack: number;
decay: number;
x: number;
y: number;
emitRange: number;
}) => void;
stopRemotePianoNote: (senderId: string, keyId: string) => void;
stopAllRemotePianoNotesForSender: (senderId: string) => void;
TELEPORT_SOUND_URL: string; TELEPORT_SOUND_URL: string;
TELEPORT_START_SOUND_URL: string; TELEPORT_START_SOUND_URL: string;
getAudioLayers: () => { world: boolean }; getAudioLayers: () => { world: boolean; item: boolean };
pushChatMessage: (message: string) => void; pushChatMessage: (message: string) => void;
classifySystemMessageSound: (message: string) => 'logon' | 'logout' | 'notify' | null; classifySystemMessageSound: (message: string) => 'logon' | 'logout' | 'notify' | null;
SYSTEM_SOUND_URLS: { logon: string; logout: string; notify: string }; SYSTEM_SOUND_URLS: { logon: string; logout: string; notify: string };
@@ -159,6 +173,7 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
if (peer) { if (peer) {
deps.updateStatus(`${peer.nickname} has left.`); deps.updateStatus(`${peer.nickname} has left.`);
} }
deps.stopAllRemotePianoNotesForSender(message.id);
deps.state.peers.delete(message.id); deps.state.peers.delete(message.id);
deps.peerManager.removePeer(message.id); deps.peerManager.removePeer(message.id);
break; break;
@@ -226,7 +241,7 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
if (message.action === 'use') { if (message.action === 'use') {
deps.pushChatMessage(message.message); deps.pushChatMessage(message.message);
const item = message.itemId ? deps.getItemById(message.itemId) : null; const item = message.itemId ? deps.getItemById(message.itemId) : null;
if (!item?.useSound && item) { if (!item?.useSound && item && item.type !== 'piano') {
deps.playLocateToneAt(item.x, item.y); deps.playLocateToneAt(item.x, item.y);
} }
} else if (message.action !== 'update') { } else if (message.action !== 'update') {
@@ -248,6 +263,27 @@ export function createOnMessageHandler(deps: MessageHandlerDeps): (message: Inco
} }
break; break;
} }
case 'item_piano_note': {
if (!deps.getAudioLayers().item) break;
if (message.on) {
deps.playRemotePianoNote({
itemId: message.itemId,
senderId: message.senderId,
keyId: message.keyId,
midi: message.midi,
instrument: message.instrument,
attack: message.attack,
decay: message.decay,
x: message.x,
y: message.y,
emitRange: message.emitRange,
});
} else {
deps.stopRemotePianoNote(message.senderId, message.keyId);
}
break;
}
} }
}; };
} }

View File

@@ -2,7 +2,7 @@ import { z } from 'zod';
export const itemSchema = z.object({ export const itemSchema = z.object({
id: z.string(), id: z.string(),
type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget']), type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget', 'piano']),
title: z.string(), title: z.string(),
x: z.number().int(), x: z.number().int(),
y: z.number().int(), y: z.number().int(),
@@ -42,10 +42,10 @@ export const welcomeMessageSchema = z.object({
.optional(), .optional(),
uiDefinitions: z uiDefinitions: z
.object({ .object({
itemTypeOrder: z.array(z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget'])), itemTypeOrder: z.array(z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget', 'piano'])),
itemTypes: z.array( itemTypes: z.array(
z.object({ z.object({
type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget']), type: z.enum(['radio_station', 'dice', 'wheel', 'clock', 'widget', 'piano']),
label: z.string().optional(), label: z.string().optional(),
tooltip: z.string().optional(), tooltip: z.string().optional(),
editableProperties: z.array(z.string()), editableProperties: z.array(z.string()),
@@ -150,6 +150,21 @@ export const itemUseSoundSchema = z.object({
y: z.number().int(), y: z.number().int(),
}); });
export const itemPianoNoteSchema = z.object({
type: z.literal('item_piano_note'),
itemId: z.string(),
senderId: z.string(),
keyId: z.string(),
midi: z.number().int().min(0).max(127),
on: z.boolean(),
instrument: z.string(),
attack: z.number().int().min(0).max(100),
decay: z.number().int().min(0).max(100),
x: z.number().int(),
y: z.number().int(),
emitRange: z.number().int().min(1),
});
export const incomingMessageSchema = z.discriminatedUnion('type', [ export const incomingMessageSchema = z.discriminatedUnion('type', [
welcomeMessageSchema, welcomeMessageSchema,
signalMessageSchema, signalMessageSchema,
@@ -163,6 +178,7 @@ export const incomingMessageSchema = z.discriminatedUnion('type', [
itemRemoveSchema, itemRemoveSchema,
itemActionResultSchema, itemActionResultSchema,
itemUseSoundSchema, itemUseSoundSchema,
itemPianoNoteSchema,
]); ]);
export type IncomingMessage = z.infer<typeof incomingMessageSchema>; export type IncomingMessage = z.infer<typeof incomingMessageSchema>;
@@ -173,11 +189,12 @@ export type OutgoingMessage =
| { type: 'update_nickname'; nickname: string } | { type: 'update_nickname'; nickname: string }
| { type: 'chat_message'; message: string } | { type: 'chat_message'; message: string }
| { type: 'ping'; clientSentAt: number } | { type: 'ping'; clientSentAt: number }
| { type: 'item_add'; itemType: 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget' } | { type: 'item_add'; itemType: 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget' | 'piano' }
| { type: 'item_pickup'; itemId: string } | { type: 'item_pickup'; itemId: string }
| { type: 'item_drop'; itemId: string; x: number; y: number } | { type: 'item_drop'; itemId: string; x: number; y: number }
| { type: 'item_delete'; itemId: string } | { type: 'item_delete'; itemId: string }
| { type: 'item_use'; itemId: string } | { type: 'item_use'; itemId: string }
| { type: 'item_piano_note'; itemId: string; keyId: string; midi: number; on: boolean }
| { | {
type: 'item_update'; type: 'item_update';
itemId: string; itemId: string;

View File

@@ -89,6 +89,8 @@ export class CanvasRenderer {
? '#fbbf24' ? '#fbbf24'
: item.type === 'wheel' : item.type === 'wheel'
? '#f97316' ? '#f97316'
: item.type === 'piano'
? '#c4b5fd'
: item.type === 'clock' : item.type === 'clock'
? '#86efac' ? '#86efac'
: item.type === 'widget' : item.type === 'widget'
@@ -103,6 +105,8 @@ export class CanvasRenderer {
? 'R' ? 'R'
: item.type === 'wheel' : item.type === 'wheel'
? 'W' ? 'W'
: item.type === 'piano'
? 'P'
: item.type === 'clock' : item.type === 'clock'
? 'C' ? 'C'
: item.type === 'widget' : item.type === 'widget'

View File

@@ -2,7 +2,7 @@ export const GRID_SIZE = 41;
export const HEARING_RADIUS = 20; export const HEARING_RADIUS = 20;
export const MOVE_COOLDOWN_MS = 200; export const MOVE_COOLDOWN_MS = 200;
export type ItemType = 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget'; export type ItemType = 'radio_station' | 'dice' | 'wheel' | 'clock' | 'widget' | 'piano';
export type WorldItem = { export type WorldItem = {
id: string; id: string;
@@ -36,7 +36,8 @@ export type GameMode =
| 'selectItem' | 'selectItem'
| 'itemProperties' | 'itemProperties'
| 'itemPropertyEdit' | 'itemPropertyEdit'
| 'itemPropertyOptionSelect'; | 'itemPropertyOptionSelect'
| 'pianoUse';
export type Player = { export type Player = {
id: string | null; id: string | null;

View File

@@ -74,6 +74,13 @@ Applies to effect select, user/item list modes, item selection, item property li
- `Space`: Read tooltip/help for current option (where metadata is available) - `Space`: Read tooltip/help for current option (where metadata is available)
- First-letter navigation: jump to next matching entry - First-letter navigation: jump to next matching entry
## Piano Use Mode
- `A S D F G H J K L ; '`: Play white keys (C major from C4 upward)
- `W E T Y U O P ]`: Play sharps
- Multiple keys can be held/played at once
- `Enter` / `Escape`: Exit piano mode
## Help Viewer Mode ## Help Viewer Mode
- `ArrowUp` / `ArrowDown`: Previous/next help line - `ArrowUp` / `ArrowDown`: Previous/next help line

View File

@@ -5,7 +5,7 @@
```json ```json
{ {
"id": "string", "id": "string",
"type": "radio_station | dice | wheel | clock | widget", "type": "radio_station | dice | wheel | clock | widget | piano",
"title": "string", "title": "string",
"x": 0, "x": 0,
"y": 0, "y": 0,
@@ -24,8 +24,8 @@
- `useSound`: optional client-played one-shot sound when item `use` succeeds; global item field and not user-editable in V1. - `useSound`: optional client-played one-shot sound when item `use` succeeds; global item field and not user-editable in V1.
- `emitSound`: optional continuously-looping spatial sound emitted from the item on the grid; global item field and not user-editable in V1. - `emitSound`: optional continuously-looping spatial sound emitted from the item on the grid; global item field and not user-editable in V1.
- `capabilities`, `useSound`, and `emitSound` are derived from global item-type definitions at runtime (not stored per-instance in persisted state). - `capabilities`, `useSound`, and `emitSound` are derived from global item-type definitions at runtime (not stored per-instance in persisted state).
- `useCooldownMs`: global per item type (`radio_station=1000`, `dice=1000`, `wheel=4000`, `clock=1000`, `widget=1000`), not per-instance editable. - `useCooldownMs`: global per item type (`radio_station=1000`, `dice=1000`, `wheel=4000`, `clock=1000`, `widget=1000`, `piano=1000`), not per-instance editable.
- `emitRange`: global spatial range default per item type (`radio_station=20`, `dice=15`, `wheel=15`, `clock=10`, `widget=15`). - `emitRange`: global spatial range default per item type (`radio_station=20`, `dice=15`, `wheel=15`, `clock=10`, `widget=15`, `piano=15`).
- `radio_station` can override this per instance via `params.emitRange` (`5..20`). - `radio_station` can override this per instance via `params.emitRange` (`5..20`).
- `directional`: global directional attenuation flag per item type (`radio_station=true`, others `false`); `widget` can override per instance via `params.directional`. - `directional`: global directional attenuation flag per item type (`radio_station=true`, others `false`); `widget` can override per instance via `params.directional`.
@@ -34,7 +34,7 @@
```json ```json
{ {
"id": "string", "id": "string",
"type": "radio_station | dice | wheel | clock | widget", "type": "radio_station | dice | wheel | clock | widget | piano",
"title": "string", "title": "string",
"x": 0, "x": 0,
"y": 0, "y": 0,
@@ -158,6 +158,23 @@
- `useSound`: empty, filename (assumed under `sounds/`), or full URL. - `useSound`: empty, filename (assumed under `sounds/`), or full URL.
- `emitSound`: empty, filename (assumed under `sounds/`), or full URL. - `emitSound`: empty, filename (assumed under `sounds/`), or full URL.
### `piano`
```json
{
"instrument": "piano",
"attack": 15,
"decay": 45,
"emitRange": 15
}
```
- `instrument`: one of
`piano | electric_piano | guitar | organ | bass | violin | synth_lead | drum_kit`.
- `attack`: integer, range `0-100`, default `15`.
- `decay`: integer, range `0-100`, default `45`.
- `emitRange`: integer, range `5-20`, default `15`.
## Packet Shapes ## Packet Shapes
- `item_upsert`: - `item_upsert`:
@@ -201,3 +218,22 @@
"y": 8 "y": 8
} }
``` ```
- `item_piano_note`:
```json
{
"type": "item_piano_note",
"itemId": "item-id",
"senderId": "user-id",
"keyId": "KeyA",
"midi": 60,
"on": true,
"instrument": "piano",
"attack": 15,
"decay": 45,
"x": 12,
"y": 8,
"emitRange": 15
}
```

View File

@@ -151,6 +151,31 @@ This is behavior-focused documentation for item types and their defaults.
- `useSound`: empty, filename (assumed under `sounds/`), or full URL - `useSound`: empty, filename (assumed under `sounds/`), or full URL
- `emitSound`: empty, filename (assumed under `sounds/`), or full URL - `emitSound`: empty, filename (assumed under `sounds/`), or full URL
## `piano`
### Defaults
- Title: `piano`
- Params:
- `instrument="piano"`
- `attack=15`
- `decay=45`
- `emitRange=15`
- Global:
- `useSound=none`
- `emitSound=none`
- `useCooldownMs=1000`
- `emitRange=15`
- `directional=false`
### Use
- Announces that the user begins playing the piano (client enters piano key mode).
### Validation
- `instrument`: `piano | electric_piano | guitar | organ | bass | violin | synth_lead | drum_kit`
- `attack`: integer `0..100`
- `decay`: integer `0..100`
- `emitRange`: integer `5..20`
## Adding A New Item Type (Registry V1) ## Adding A New Item Type (Registry V1)
Item types are currently code-registered on both server and client. Server item logic is split per item module and wired through one registry. Item types are currently code-registered on both server and client. Server item logic is split per item module and wired through one registry.

View File

@@ -15,6 +15,7 @@ This is a behavior guide for packet semantics beyond raw schemas.
- `chat_message`: player chat. - `chat_message`: player chat.
- `ping`: latency measurement. - `ping`: latency measurement.
- `item_add`, `item_pickup`, `item_drop`, `item_delete`, `item_use`, `item_update`: item actions. - `item_add`, `item_pickup`, `item_drop`, `item_delete`, `item_use`, `item_update`: item actions.
- `item_piano_note`: realtime piano note on/off for active piano use mode.
## Server -> Client ## Server -> Client
@@ -28,12 +29,17 @@ This is a behavior guide for packet semantics beyond raw schemas.
- `item_remove`: item deletion. - `item_remove`: item deletion.
- `item_action_result`: action success/failure and user-facing message. - `item_action_result`: action success/failure and user-facing message.
- `item_use_sound`: spatial one-shot sound on successful item use (if `useSound` configured). - `item_use_sound`: spatial one-shot sound on successful item use (if `useSound` configured).
- `item_piano_note`: broadcast piano note on/off with resolved instrument/envelope/spatial params.
## Item Packet Behavior ## Item Packet Behavior
- `item_upsert` is full-state replacement for one item, not partial patch. - `item_upsert` is full-state replacement for one item, not partial patch.
- `item_action_result` messages are intended for direct screen-reader/user status feedback. - `item_action_result` messages are intended for direct screen-reader/user status feedback.
- `item_use_sound` contains absolute item world coordinates (`x`, `y`) and sound path. - `item_use_sound` contains absolute item world coordinates (`x`, `y`) and sound path.
- `item_piano_note` contains:
- `itemId`, `senderId`, `keyId`, `midi`, `on`
- resolved `instrument`, `attack`, `decay`, `emitRange`
- absolute source coordinates `x`, `y`
## Welcome Metadata ## Welcome Metadata

View File

@@ -41,6 +41,7 @@ Core incoming message effects:
- `item_remove`: remove item and cleanup runtimes. - `item_remove`: remove item and cleanup runtimes.
- `item_action_result`: success/error status for actions. - `item_action_result`: success/error status for actions.
- `item_use_sound`: play one-shot spatial sample (world layer gated). - `item_use_sound`: play one-shot spatial sample (world layer gated).
- `item_piano_note`: start/stop synthesized piano notes from remote users (item layer gated).
- `pong`: - `pong`:
- positive `clientSentAt`: user ping response (`P` command) - positive `clientSentAt`: user ping response (`P` command)
- negative `clientSentAt`: internal heartbeat response - negative `clientSentAt`: internal heartbeat response

View File

@@ -5,10 +5,10 @@ from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from typing import Literal, cast from typing import Literal, cast
from .items import clock, radio from .items import clock, piano, radio
from .items.registry import ITEM_MODULES, ITEM_TYPE_ORDER from .items.registry import ITEM_MODULES, ITEM_TYPE_ORDER
ItemType = Literal["radio_station", "dice", "wheel", "clock", "widget"] ItemType = Literal["radio_station", "dice", "wheel", "clock", "widget", "piano"]
ITEM_TYPE_SEQUENCE: tuple[ItemType, ...] = cast(tuple[ItemType, ...], ITEM_TYPE_ORDER) ITEM_TYPE_SEQUENCE: tuple[ItemType, ...] = cast(tuple[ItemType, ...], ITEM_TYPE_ORDER)
ITEM_TYPE_LABELS: dict[ItemType, str] = {item_type: ITEM_MODULES[item_type].LABEL for item_type in ITEM_TYPE_SEQUENCE} ITEM_TYPE_LABELS: dict[ItemType, str] = {item_type: ITEM_MODULES[item_type].LABEL for item_type in ITEM_TYPE_SEQUENCE}
ITEM_TYPE_EDITABLE_PROPERTIES: dict[ItemType, tuple[str, ...]] = { ITEM_TYPE_EDITABLE_PROPERTIES: dict[ItemType, tuple[str, ...]] = {
@@ -19,6 +19,7 @@ CLOCK_DEFAULT_TIME_ZONE = clock.DEFAULT_TIME_ZONE
CLOCK_TIME_ZONE_OPTIONS = clock.TIME_ZONE_OPTIONS CLOCK_TIME_ZONE_OPTIONS = clock.TIME_ZONE_OPTIONS
RADIO_EFFECT_OPTIONS = radio.EFFECT_OPTIONS RADIO_EFFECT_OPTIONS = radio.EFFECT_OPTIONS
RADIO_CHANNEL_OPTIONS = radio.CHANNEL_OPTIONS RADIO_CHANNEL_OPTIONS = radio.CHANNEL_OPTIONS
PIANO_INSTRUMENT_OPTIONS = piano.INSTRUMENT_OPTIONS
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -79,6 +80,7 @@ ITEM_PROPERTY_OPTIONS: dict[str, tuple[str, ...]] = {
"emitEffect": RADIO_EFFECT_OPTIONS, "emitEffect": RADIO_EFFECT_OPTIONS,
"mediaChannel": RADIO_CHANNEL_OPTIONS, "mediaChannel": RADIO_CHANNEL_OPTIONS,
"timeZone": CLOCK_TIME_ZONE_OPTIONS, "timeZone": CLOCK_TIME_ZONE_OPTIONS,
"instrument": PIANO_INSTRUMENT_OPTIONS,
} }
ITEM_TYPE_TOOLTIPS: dict[ItemType, str] = { ITEM_TYPE_TOOLTIPS: dict[ItemType, str] = {

View File

@@ -33,7 +33,7 @@ class ItemService:
return int(time.time() * 1000) return int(time.time() * 1000)
def default_item(self, client: ClientConnection, item_type: Literal["radio_station", "dice", "wheel", "clock", "widget"]) -> WorldItem: def default_item(self, client: ClientConnection, item_type: Literal["radio_station", "dice", "wheel", "clock", "widget", "piano"]) -> WorldItem:
"""Create a new server-authoritative item at the caller's position.""" """Create a new server-authoritative item at the caller's position."""
item_def = get_item_definition(item_type) item_def = get_item_definition(item_type)

100
server/app/items/piano.py Normal file
View File

@@ -0,0 +1,100 @@
"""Piano item schema metadata and behavior."""
from __future__ import annotations
from typing import Callable
from ..item_types import ItemUseResult
from ..models import WorldItem
LABEL = "piano"
TOOLTIP = "Playable keyboard instrument with multiple synth voices."
EDITABLE_PROPERTIES: tuple[str, ...] = ("title", "instrument", "attack", "decay", "emitRange")
CAPABILITIES: tuple[str, ...] = ("editable", "carryable", "deletable", "usable")
USE_SOUND: str | None = None
EMIT_SOUND: str | None = None
USE_COOLDOWN_MS = 1000
EMIT_RANGE = 15
DIRECTIONAL = False
DEFAULT_TITLE = "piano"
DEFAULT_PARAMS: dict = {
"instrument": "piano",
"attack": 15,
"decay": 45,
"emitRange": 15,
}
INSTRUMENT_OPTIONS: tuple[str, ...] = (
"piano",
"electric_piano",
"guitar",
"organ",
"bass",
"violin",
"synth_lead",
"drum_kit",
)
PROPERTY_METADATA: dict[str, dict[str, object]] = {
"title": {"valueType": "text", "tooltip": "Display name spoken and shown for this item.", "maxLength": 80},
"instrument": {"valueType": "list", "tooltip": "Instrument voice used when playing this piano."},
"attack": {
"valueType": "number",
"tooltip": "How quickly notes ramp in. Lower is sharper; higher is softer.",
"range": {"min": 0, "max": 100, "step": 1},
},
"decay": {
"valueType": "number",
"tooltip": "How long notes ring out after the initial hit.",
"range": {"min": 0, "max": 100, "step": 1},
},
"emitRange": {
"valueType": "number",
"tooltip": "Maximum distance in squares where this piano can be heard.",
"range": {"min": 5, "max": 20, "step": 1},
},
}
def validate_update(_item: WorldItem, next_params: dict) -> dict:
"""Validate and normalize piano params."""
instrument = str(next_params.get("instrument", "piano")).strip().lower()
if instrument not in INSTRUMENT_OPTIONS:
raise ValueError(f"instrument must be one of: {', '.join(INSTRUMENT_OPTIONS)}.")
next_params["instrument"] = instrument
try:
attack = int(next_params.get("attack", 15))
except (TypeError, ValueError) as exc:
raise ValueError("attack must be an integer between 0 and 100.") from exc
if not (0 <= attack <= 100):
raise ValueError("attack must be between 0 and 100.")
next_params["attack"] = attack
try:
decay = int(next_params.get("decay", 45))
except (TypeError, ValueError) as exc:
raise ValueError("decay must be an integer between 0 and 100.") from exc
if not (0 <= decay <= 100):
raise ValueError("decay must be between 0 and 100.")
next_params["decay"] = decay
try:
emit_range = int(next_params.get("emitRange", 15))
except (TypeError, ValueError) as exc:
raise ValueError("emitRange must be an integer between 5 and 20.") from exc
if not (5 <= emit_range <= 20):
raise ValueError("emitRange must be between 5 and 20.")
next_params["emitRange"] = emit_range
return next_params
def use_item(item: WorldItem, nickname: str, _clock_formatter: Callable[[dict], str]) -> ItemUseResult:
"""Enter piano play mode for the user who used the item."""
return ItemUseResult(
self_message=f"You begin playing {item.title}.",
others_message=f"{nickname} begins playing {item.title}.",
)

View File

@@ -7,7 +7,7 @@ from typing import Callable, Protocol
from ..item_types import ItemUseResult from ..item_types import ItemUseResult
from ..models import WorldItem from ..models import WorldItem
from . import clock, dice, radio, wheel, widget from . import clock, dice, piano, radio, wheel, widget
class ItemModule(Protocol): class ItemModule(Protocol):
@@ -29,11 +29,12 @@ class ItemModule(Protocol):
use_item: Callable[[WorldItem, str, Callable[[dict], str]], ItemUseResult] use_item: Callable[[WorldItem, str, Callable[[dict], str]], ItemUseResult]
ITEM_TYPE_ORDER: tuple[str, ...] = ("clock", "dice", "radio_station", "wheel", "widget") ITEM_TYPE_ORDER: tuple[str, ...] = ("clock", "dice", "piano", "radio_station", "wheel", "widget")
ITEM_MODULES: dict[str, ItemModule] = { ITEM_MODULES: dict[str, ItemModule] = {
"clock": clock, "clock": clock,
"dice": dice, "dice": dice,
"piano": piano,
"radio_station": radio, "radio_station": radio,
"wheel": wheel, "wheel": wheel,
"widget": widget, "widget": widget,

View File

@@ -42,7 +42,7 @@ class PingPacket(BasePacket):
class ItemAddPacket(BasePacket): class ItemAddPacket(BasePacket):
type: Literal["item_add"] type: Literal["item_add"]
itemType: Literal["radio_station", "dice", "wheel", "clock", "widget"] itemType: Literal["radio_station", "dice", "wheel", "clock", "widget", "piano"]
class ItemPickupPacket(BasePacket): class ItemPickupPacket(BasePacket):
@@ -67,6 +67,14 @@ class ItemUsePacket(BasePacket):
itemId: str itemId: str
class ItemPianoNotePacket(BasePacket):
type: Literal["item_piano_note"]
itemId: str
keyId: str = Field(min_length=1, max_length=32)
midi: int = Field(ge=0, le=127)
on: bool
class ItemUpdatePacket(BasePacket): class ItemUpdatePacket(BasePacket):
type: Literal["item_update"] type: Literal["item_update"]
itemId: str itemId: str
@@ -85,6 +93,7 @@ ClientPacket = (
| ItemDropPacket | ItemDropPacket
| ItemDeletePacket | ItemDeletePacket
| ItemUsePacket | ItemUsePacket
| ItemPianoNotePacket
| ItemUpdatePacket | ItemUpdatePacket
) )
@@ -157,7 +166,7 @@ class NicknameResultPacket(BasePacket):
class WorldItem(BaseModel): class WorldItem(BaseModel):
id: str id: str
type: Literal["radio_station", "dice", "wheel", "clock", "widget"] type: Literal["radio_station", "dice", "wheel", "clock", "widget", "piano"]
title: str title: str
x: int x: int
y: int y: int
@@ -175,7 +184,7 @@ class WorldItem(BaseModel):
class PersistedWorldItem(BaseModel): class PersistedWorldItem(BaseModel):
model_config = ConfigDict(extra="ignore") model_config = ConfigDict(extra="ignore")
id: str id: str
type: Literal["radio_station", "dice", "wheel", "clock", "widget"] type: Literal["radio_station", "dice", "wheel", "clock", "widget", "piano"]
title: str title: str
x: int x: int
y: int y: int
@@ -211,3 +220,18 @@ class ItemUseSoundPacket(BasePacket):
sound: str sound: str
x: int x: int
y: int y: int
class ItemPianoNoteBroadcastPacket(BasePacket):
type: Literal["item_piano_note"]
itemId: str
senderId: str
keyId: str
midi: int
on: bool
instrument: str
attack: int
decay: int
x: int
y: int
emitRange: int

View File

@@ -46,6 +46,8 @@ from .models import (
ItemAddPacket, ItemAddPacket,
ItemDeletePacket, ItemDeletePacket,
ItemDropPacket, ItemDropPacket,
ItemPianoNoteBroadcastPacket,
ItemPianoNotePacket,
ItemPickupPacket, ItemPickupPacket,
ItemRemovePacket, ItemRemovePacket,
ItemUpdatePacket, ItemUpdatePacket,
@@ -656,6 +658,39 @@ class SignalingServer:
) )
return return
if isinstance(packet, ItemPianoNotePacket):
item = self.items.get(packet.itemId)
if not item or item.type != "piano":
return
if item.carrierId not in (None, client.id):
return
if item.carrierId is None and (item.x != client.x or item.y != client.y):
return
instrument = str(item.params.get("instrument", "piano")).strip().lower()
attack = int(item.params.get("attack", 15)) if isinstance(item.params.get("attack", 15), (int, float)) else 15
decay = int(item.params.get("decay", 45)) if isinstance(item.params.get("decay", 45), (int, float)) else 45
emit_range = int(item.params.get("emitRange", 15)) if isinstance(item.params.get("emitRange", 15), (int, float)) else 15
source_x = client.x if item.carrierId == client.id else item.x
source_y = client.y if item.carrierId == client.id else item.y
await self._broadcast(
ItemPianoNoteBroadcastPacket(
type="item_piano_note",
itemId=item.id,
senderId=client.id,
keyId=packet.keyId,
midi=packet.midi,
on=packet.on,
instrument=instrument,
attack=max(0, min(100, attack)),
decay=max(0, min(100, decay)),
x=source_x,
y=source_y,
emitRange=max(5, min(20, emit_range)),
),
exclude=client.websocket,
)
return
if isinstance(packet, ItemUpdatePacket): if isinstance(packet, ItemUpdatePacket):
item = self.items.get(packet.itemId) item = self.items.get(packet.itemId)
if not item: if not item:

View File

@@ -339,3 +339,102 @@ async def test_widget_update_and_use(monkeypatch: pytest.MonkeyPatch) -> None:
) )
assert send_payloads[-1].ok is False assert send_payloads[-1].ok is False
assert "emitsoundtempo must be between 0 and 100" in send_payloads[-1].message.lower() assert "emitsoundtempo must be between 0 and 100" in send_payloads[-1].message.lower()
@pytest.mark.asyncio
async def test_piano_update_and_use(monkeypatch: pytest.MonkeyPatch) -> None:
server = SignalingServer("127.0.0.1", 8765, None, None)
ws = _fake_ws()
client = ClientConnection(websocket=ws, id="u1", nickname="tester", x=5, y=6)
server.clients[ws] = client
item = server.item_service.default_item(client, "piano")
server.item_service.add_item(item)
send_payloads: list[object] = []
broadcast_payloads: list[object] = []
async def fake_send(websocket: ServerConnection, packet: object) -> None:
send_payloads.append(packet)
async def fake_broadcast(packet: object, exclude: ServerConnection | None = None) -> None:
broadcast_payloads.append(packet)
monkeypatch.setattr(server, "_send", fake_send)
monkeypatch.setattr(server, "_broadcast", fake_broadcast)
await server._handle_message(
client,
json.dumps(
{
"type": "item_update",
"itemId": item.id,
"params": {
"instrument": "drum_kit",
"attack": 22,
"decay": 67,
"emitRange": 12,
},
}
),
)
assert send_payloads[-1].ok is True
assert item.params.get("instrument") == "drum_kit"
assert item.params.get("attack") == 22
assert item.params.get("decay") == 67
assert item.params.get("emitRange") == 12
await server._handle_message(client, json.dumps({"type": "item_use", "itemId": item.id}))
assert send_payloads[-1].ok is True
assert "begin playing" in send_payloads[-1].message.lower()
assert not any(getattr(packet, "type", "") == "item_use_sound" for packet in broadcast_payloads)
await server._handle_message(
client,
json.dumps({"type": "item_update", "itemId": item.id, "params": {"instrument": "banjo"}}),
)
assert send_payloads[-1].ok is False
assert "instrument must be one of" in send_payloads[-1].message.lower()
@pytest.mark.asyncio
async def test_piano_note_packet_broadcasts(monkeypatch: pytest.MonkeyPatch) -> None:
server = SignalingServer("127.0.0.1", 8765, None, None)
ws_sender = _fake_ws()
sender = ClientConnection(websocket=ws_sender, id="u1", nickname="tester", x=5, y=6)
ws_other = _fake_ws()
other = ClientConnection(websocket=ws_other, id="u2", nickname="listener", x=7, y=6)
server.clients[ws_sender] = sender
server.clients[ws_other] = other
item = server.item_service.default_item(sender, "piano")
item.params["instrument"] = "organ"
item.params["attack"] = 20
item.params["decay"] = 60
item.params["emitRange"] = 12
server.item_service.add_item(item)
send_payloads: list[object] = []
broadcast_payloads: list[object] = []
async def fake_send(websocket: ServerConnection, packet: object) -> None:
send_payloads.append(packet)
async def fake_broadcast(packet: object, exclude: ServerConnection | None = None) -> None:
broadcast_payloads.append(packet)
monkeypatch.setattr(server, "_send", fake_send)
monkeypatch.setattr(server, "_broadcast", fake_broadcast)
await server._handle_message(
sender,
json.dumps({"type": "item_piano_note", "itemId": item.id, "keyId": "KeyA", "midi": 60, "on": True}),
)
assert not send_payloads
assert broadcast_payloads
packet = broadcast_payloads[-1]
assert getattr(packet, "type", "") == "item_piano_note"
assert getattr(packet, "itemId", "") == item.id
assert getattr(packet, "instrument", "") == "organ"
assert getattr(packet, "attack", -1) == 20
assert getattr(packet, "decay", -1) == 60
assert getattr(packet, "emitRange", -1) == 12

View File

@@ -16,3 +16,9 @@ def test_unknown_type_rejected() -> None:
except ValidationError: except ValidationError:
return return
assert False, "validation should fail" assert False, "validation should fail"
def test_item_add_accepts_piano_type() -> None:
adapter = TypeAdapter(ClientPacket)
packet = adapter.validate_python({"type": "item_add", "itemType": "piano"})
assert packet.type == "item_add"