feat: add speech visualizer and emotion-driven audio cues

- Introduce a canvas-based audio visualizer overlay on opponent portraits that activates during speech playback
- Update `SpeechQueue` to support `onStart` and `onEnd` callbacks for hooking into audio events
- Modify `Portrait` to trigger the visualizer with emotion-specific colors (blue for intro, green for happy, red for upset)
- Add `speech` metadata to `opponents.json` for Ethel and Bernie, defining available intro/happy/upset audio clips
- Ensure proper cleanup of visualizer state and resources on portrait destruction
This commit is contained in:
Brian Fertig 2026-05-22 19:10:36 -06:00
parent f6a4a3fb5d
commit 3fcfbcc921
29 changed files with 140 additions and 10 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -94,7 +94,27 @@
"id": "ethel", "id": "ethel",
"spriteIndex": 5, "spriteIndex": 5,
"name": "Ethel", "name": "Ethel",
"bio": "Thank you for visiting with me." "bio": "Thank you for visiting with me.",
"speech": {
"intro": [
"ethel-intro-01",
"ethel-intro-02"
],
"happy": [
"ethel-happy-01",
"ethel-happy-02",
"ethel-happy-03",
"ethel-happy-04",
"ethel-happy-05"
],
"upset": [
"ethel-upset-01",
"ethel-upset-02",
"ethel-upset-03",
"ethel-upset-04",
"ethel-upset-05"
]
}
}, },
{ {
"id": "jeff", "id": "jeff",
@ -118,7 +138,20 @@
"id": "bernie", "id": "bernie",
"spriteIndex": 9, "spriteIndex": 9,
"name": "Bernie", "name": "Bernie",
"bio": "Having fun and playing is the name of the game for me!" "bio": "Having fun and playing is the name of the game for me!",
"speech": {
"intro": [
"bernie-intro-01"
],
"happy": [
"bernie-happy-01",
"bernie-happy-02"
],
"upset": [
"bernie-upset-01",
"bernie-upset-02"
]
}
} }
] ]
} }

View File

@ -75,6 +75,78 @@ export function createOpponentPortrait(scene, opponent, worldX, worldY, radius,
const domEl = scene.add.dom(worldX, worldY, videoEl).setDepth(depth + 2); const domEl = scene.add.dom(worldX, worldY, videoEl).setDepth(depth + 2);
// Speech visualizer canvas — overlays the portrait when audio is playing
const canvasEl = document.createElement('canvas');
canvasEl.width = size;
canvasEl.height = size;
canvasEl.style.cssText = `width:${size}px;height:${size}px;border-radius:50%;pointer-events:none;opacity:0;transition:opacity 0.2s ease;`;
const canvasDom = scene.add.dom(worldX, worldY, canvasEl).setDepth(depth + 3);
const ctx = canvasEl.getContext('2d');
const BAR_COUNT = 6;
const bars = Array.from({ length: BAR_COUNT }, () => ({ cur: 0.1, target: 0.5 }));
let vizActive = false;
let rafId = null;
let retargetTimer = null;
let vizColor = [0, 200, 255];
function _vizFrame() {
if (!vizActive) { rafId = null; ctx.clearRect(0, 0, size, size); return; }
ctx.clearRect(0, 0, size, size);
const gy = size * 0.58;
const grad = ctx.createLinearGradient(0, gy, 0, size);
grad.addColorStop(0, 'rgba(0,0,0,0)');
grad.addColorStop(1, 'rgba(0,0,0,0.38)');
ctx.fillStyle = grad;
ctx.fillRect(0, gy, size, size - gy);
const [r, g, b] = vizColor;
const barW = size * 0.072;
const gap = size * 0.03;
const totalW = BAR_COUNT * barW + (BAR_COUNT - 1) * gap;
const startX = (size - totalW) / 2;
const maxH = size * 0.36;
const baseY = size * 0.88;
bars.forEach((bar, i) => {
bar.cur += (bar.target - bar.cur) * 0.14;
const h = Math.max(bar.cur * maxH, size * 0.04);
const x = startX + i * (barW + gap);
ctx.shadowColor = `rgba(${r},${g},${b},0.7)`;
ctx.shadowBlur = 5;
ctx.fillStyle = `rgba(${r},${g},${b},0.88)`;
ctx.beginPath();
ctx.roundRect(x, baseY - h, barW, h, 2);
ctx.fill();
});
ctx.shadowBlur = 0;
rafId = requestAnimationFrame(_vizFrame);
}
function startVisualizer(emotion) {
if (emotion === 'happy') vizColor = [0, 210, 60];
else if (emotion === 'upset') vizColor = [210, 60, 60];
else vizColor = [0, 200, 255];
vizActive = true;
canvasEl.style.opacity = '1';
if (!retargetTimer) {
retargetTimer = setInterval(() => {
bars.forEach(b => { b.target = 0.2 + Math.random() * 0.8; });
}, 120);
}
if (!rafId) rafId = requestAnimationFrame(_vizFrame);
}
function stopVisualizer() {
vizActive = false;
canvasEl.style.opacity = '0';
clearInterval(retargetTimer);
retargetTimer = null;
}
let emotionPlaying = false; let emotionPlaying = false;
function playEmotion(emotion) { function playEmotion(emotion) {
@ -101,7 +173,10 @@ export function createOpponentPortrait(scene, opponent, worldX, worldY, radius,
const speechClips = opponent?.speech?.[emotion]; const speechClips = opponent?.speech?.[emotion];
if (speechClips?.length && Math.random() < 0.6) { if (speechClips?.length && Math.random() < 0.6) {
enqueueSpeech(speechClips[Math.floor(Math.random() * speechClips.length)]); enqueueSpeech(speechClips[Math.floor(Math.random() * speechClips.length)], {
onStart: () => startVisualizer(emotion),
onEnd: stopVisualizer,
});
} }
} }
@ -121,13 +196,21 @@ export function createOpponentPortrait(scene, opponent, worldX, worldY, radius,
videoEl.pause(); videoEl.pause();
videoEl.src = ''; videoEl.src = '';
resetSpeechQueue(); resetSpeechQueue();
vizActive = false;
if (rafId) { cancelAnimationFrame(rafId); rafId = null; }
clearInterval(retargetTimer);
retargetTimer = null;
canvasDom.destroy();
} }
scene.events.once('shutdown', destroy); scene.events.once('shutdown', destroy);
if (opponent?.speech?.intro?.length) { if (opponent?.speech?.intro?.length) {
const clips = opponent.speech.intro; const clips = opponent.speech.intro;
enqueueSpeech(clips[Math.floor(Math.random() * clips.length)]); enqueueSpeech(clips[Math.floor(Math.random() * clips.length)], {
onStart: () => startVisualizer('intro'),
onEnd: stopVisualizer,
});
} }
return { playEmotion, hide, show, destroy }; return { playEmotion, hide, show, destroy };

View File

@ -1,27 +1,41 @@
let _queue = []; let _queue = [];
let _playing = false; let _playing = false;
let _currentAudio = null; let _currentAudio = null;
let _currentCbs = null;
const MAX = 4; const MAX = 4;
export function enqueue(filename) { export function enqueue(filename, callbacks) {
if (_queue.length + (_playing ? 1 : 0) >= MAX) return; if (_queue.length + (_playing ? 1 : 0) >= MAX) return;
_queue.push(filename); _queue.push({ filename, onStart: callbacks?.onStart, onEnd: callbacks?.onEnd });
_playNext(); _playNext();
} }
function _playNext() { function _playNext() {
if (_playing || !_queue.length) return; if (_playing || !_queue.length) return;
_playing = true; _playing = true;
_currentAudio = new Audio(`/assets/speech/${_queue.shift()}.mp3`); const entry = _queue.shift();
_currentCbs = entry;
_currentAudio = new Audio(`/assets/speech/${entry.filename}.mp3`);
_currentAudio.volume = 0.8; _currentAudio.volume = 0.8;
const done = () => { _playing = false; _currentAudio = null; _playNext(); }; const done = () => {
entry.onEnd?.();
_playing = false;
_currentAudio = null;
_currentCbs = null;
_playNext();
};
_currentAudio.onended = done; _currentAudio.onended = done;
_currentAudio.onerror = done; _currentAudio.onerror = done;
_currentAudio.play().catch(done); _currentAudio.play().then(() => entry.onStart?.()).catch(done);
} }
export function resetQueue() { export function resetQueue() {
if (_currentAudio) { _currentAudio.pause(); _currentAudio = null; } if (_currentAudio) {
_currentCbs?.onEnd?.();
_currentAudio.pause();
_currentAudio = null;
}
_currentCbs = null;
_queue = []; _queue = [];
_playing = false; _playing = false;
} }

0
server/db/game.db Normal file
View File