From 26351e244d6b101b1a3b66ba39ca21b1f034ace7 Mon Sep 17 00:00:00 2001 From: OmikronApex Date: Tue, 14 Jul 2026 22:51:34 +0200 Subject: [PATCH] feat(audio): stems ride a bespoke mixer channel; tuner/minigames use capture demands Ownership-plan client migrations (feedBack-desktop feat/audio-ownership-phase-a counterpart): - juce-audio.js: the stems graph requests its own mixer channel (audio.mixer.requestChannel('stems')) and pushes there instead of the aggregate renderer bus - a renderer stall no longer starves stem audio behind everything else, and per-channel underflow counters name it in field logs. Sink already flips to 'none' before any push, so the s5 double-audio guard holds by construction. Legacy renderer-bus path kept for old desktop mains / no-capacity refusals. - tuner + minigames: startAudio start-then-remember-to-undo hacks replaced by the refcounted 'capture' demand (plan s6.1) with legacy fallback; demand released on stop/supersede, and holder death releases it anyway. Co-Authored-By: Claude Fable 5 --- plugins/minigames/screen.js | 15 +++++++++-- plugins/tuner/utils/audio.js | 38 ++++++++++++++++++++------- static/js/juce-audio.js | 51 +++++++++++++++++++++++++++++++++--- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/plugins/minigames/screen.js b/plugins/minigames/screen.js index 66bbb00..949098d 100644 --- a/plugins/minigames/screen.js +++ b/plugins/minigames/screen.js @@ -135,6 +135,7 @@ // post-noise-gate frames from the engine instead — same device the player // and note_detect read from. getUserMedia stays as the web fallback. let bridgePoll = null; // setTimeout handle for the engine poll loop + let holdsCaptureDemand = false; // engine `capture` demand held (plan §6.1) let usingBridge = false; let bridgeGotFrame = false; // first non-empty frame seen (vs downlevel addon) let bridgeSampleRate = 48000; // queried once from the engine @@ -268,6 +269,10 @@ stopped = true; if (bridgePoll) { try { clearTimeout(bridgePoll); } catch (e) {} bridgePoll = null; } usingBridge = false; + if (holdsCaptureDemand) { + holdsCaptureDemand = false; + try { window.feedBackDesktop?.audio?.leases?.releaseDemand('capture', 'minigames'); } catch (e) {} + } try { if (processor) processor.disconnect(); } catch (e) {} try { if (source) source.disconnect(); } catch (e) {} try { if (mediaStream) mediaStream.getTracks().forEach(t => t.stop()); } catch (e) {} @@ -285,8 +290,14 @@ // is downlevel (getRawAudioFrame resolves an empty array). async function startBridge(audio) { try { - // The minigame needs live input; make sure the engine is capturing. - if (typeof audio.isAudioRunning === 'function') { + // The minigame needs live input. Prefer the refcounted capture + // demand (ownership plan §6.1): the engine runs while any holder + // needs it, and a dead renderer releases automatically — no + // start-then-remember-to-undo. Legacy raw start on old mains. + if (typeof audio.leases?.acquireDemand === 'function') { + await audio.leases.acquireDemand('capture', 'minigames'); + holdsCaptureDemand = true; + } else if (typeof audio.isAudioRunning === 'function') { const running = await audio.isAudioRunning(); if (!running && typeof audio.startAudio === 'function') await audio.startAudio(); } diff --git a/plugins/tuner/utils/audio.js b/plugins/tuner/utils/audio.js index 970ea3d..4cfc9c5 100644 --- a/plugins/tuner/utils/audio.js +++ b/plugins/tuner/utils/audio.js @@ -28,6 +28,14 @@ // if it changes mid-flight, so a re-entrant start()/restart() can't orphan // a worker + interval created by a superseded call. let _startGen = 0; + // Whether we hold the engine `capture` demand (ownership plan §6.1). + let _holdsCaptureDemand = false; + + function _releaseCaptureDemand(desktop) { + if (!_holdsCaptureDemand) return; + _holdsCaptureDemand = false; + try { desktop?.audio?.leases?.releaseDemand('capture', 'tuner'); } catch (_) {} + } // Timestamp of the last frame posted to the worker (watchdog, see above). let _frameSentAt = 0; @@ -104,20 +112,31 @@ var started = false; try { - var running = typeof desktop.audio.isAudioRunning === 'function' - ? await desktop.audio.isAudioRunning() : false; - if (!running && typeof desktop.audio.startAudio === 'function') { - await desktop.audio.startAudio(); - started = true; + if (typeof desktop.audio.leases?.acquireDemand === 'function') { + // Ownership plan §6.1: express "the engine must capture" as a + // refcounted demand instead of the start-then-remember-to-undo + // hack — the engine keeps running while ANY holder needs it, + // and a dead renderer releases automatically. + await desktop.audio.leases.acquireDemand('capture', 'tuner'); + _holdsCaptureDemand = true; + } else { + // Legacy desktop main without the lease registry. + var running = typeof desktop.audio.isAudioRunning === 'function' + ? await desktop.audio.isAudioRunning() : false; + if (!running && typeof desktop.audio.startAudio === 'function') { + await desktop.audio.startAudio(); + started = true; + } } } catch (e) { - // A failed startAudio means frames will never arrive — surface it + // A failed start/demand means frames will never arrive — surface it // rather than silently claiming a dead bridge. - console.warn('[tuner] bridge startAudio failed:', e && e.message ? e.message : e); + console.warn('[tuner] bridge capture request failed:', e && e.message ? e.message : e); } if (myGen !== _startGen) { - // Superseded by a newer start/stop while awaiting — undo any engine - // start we triggered and bail without claiming the bridge. + // Superseded by a newer start/stop while awaiting — undo whatever + // we acquired and bail without claiming the bridge. + _releaseCaptureDemand(desktop); if (started && typeof desktop.audio.stopAudio === 'function') { try { desktop.audio.stopAudio(); } catch (_) {} } @@ -249,6 +268,7 @@ // Invalidate any start still suspended on an await so it aborts instead // of installing a worker/interval after we've torn down. _startGen++; + _releaseCaptureDemand((typeof window !== 'undefined') ? window.feedBackDesktop : null); if (_bridgeInterval) { clearInterval(_bridgeInterval); _bridgeInterval = null; } _usingDesktopBridge = false; if (_detectInterval) { clearInterval(_detectInterval); _detectInterval = null; } diff --git a/static/js/juce-audio.js b/static/js/juce-audio.js index f05f8a7..3205497 100644 --- a/static/js/juce-audio.js +++ b/static/js/juce-audio.js @@ -535,7 +535,13 @@ import { S } from './player-state.js'; // One tap per captured graph. `active` gates the push (the worklet keeps // running when inactive — it's silent bookkeeping, not audio). function _makeTap(ctx) { - const state = { node: null, active: false, batch: [], batchFrames: 0 }; + // state.push is the producer sink: the legacy renderer-bus push by + // default; the stems engage repoints it at that session's bespoke + // mixer channel (ownership plan §5.1 tier 3). Settable per engage + // because the tap object is cached per context (_stemsTaps) while + // channel ids change per request. + const state = { node: null, active: false, batch: [], batchFrames: 0, + push: (buf, rate) => api.pushRendererAudio(buf, rate) }; state.attach = async (sourceNode) => { if (!_tapModuleLoaded.has(ctx)) { await ctx.audioWorklet.addModule(_tapModuleUrl); @@ -552,7 +558,7 @@ import { S } from './player-state.js'; const merged = new Float32Array(state.batchFrames * 2); let o = 0; for (const c of state.batch) { merged.set(c, o); o += c.length; } - api.pushRendererAudio(merged, ctx.sampleRate); + state.push(merged, ctx.sampleRate); state.batch = []; state.batchFrames = 0; } }; @@ -713,10 +719,26 @@ import { S } from './player-state.js'; } } + // Bespoke mixer channel for the stems session (ownership plan §5.1 tier 3, + // §8 — the direct tester payoff: stem audio stops riding the aggregate + // channel #0, so a renderer stall can't starve it behind everything else + // and per-channel underflow counters name it in field logs). Falls back to + // the legacy renderer-bus path on an old desktop main without audio.mixer. + let _stemsChannelId = null; + + async function _releaseStemsChannel() { + if (_stemsChannelId == null) return; + const id = _stemsChannelId; + _stemsChannelId = null; + try { await api.mixer.releaseChannel(id); } catch (_) { /* engine gone */ } + } + async function _disengage() { if (_mode === 'off') return; const prev = _mode; _mode = 'off'; + // Stems on a bespoke channel never enabled the renderer bus; the + // other modes did. Disable is harmless either way (fail-soft). try { await api.setRendererBus(false, 0); } catch (_) { /* engine gone */ } if (prev === 'loopback') { await _teardownLoopback(); @@ -725,6 +747,7 @@ import { S } from './player-state.js'; await _setSink(_elCtx, false).catch(() => {}); } else if (prev === 'stems' && _stemsGraph) { if (_stemsTap) _stemsTap.detach(_stemsGraph.masterNode); + await _releaseStemsChannel(); await _setSink(_stemsGraph.context, false).catch(() => {}); _stemsGraph = null; _stemsTap = null; } @@ -732,15 +755,35 @@ import { S } from './player-state.js'; } async function _engageStems(graph) { + // Sink goes to 'none' FIRST — the graph is off the renderer master + // before any push, so the §5 double-audio guard holds by construction. await _setSink(graph.context, true); let tap = _stemsTaps.get(graph.context); if (!tap) { tap = _makeTap(graph.context); _stemsTaps.set(graph.context, tap); } await tap.attach(graph.masterNode); - await api.setRendererBus(true, 1.0); + + let viaChannel = false; + if (typeof api.mixer?.requestChannel === 'function') { + try { + const res = await api.mixer.requestChannel('stems', 'stems'); + if (res && typeof res.channelId === 'number' && res.channelId > 0) { + _stemsChannelId = res.channelId; + const id = res.channelId; + tap.push = (buf, rate) => api.mixer.push(id, buf, rate); + viaChannel = true; + } + } catch (_) { /* fall through to legacy */ } + } + if (!viaChannel) { + // Legacy path (old desktop main, or channel refused: no-capacity). + tap.push = (buf, rate) => api.pushRendererAudio(buf, rate); + await api.setRendererBus(true, 1.0); + } tap.active = true; _stemsGraph = graph; _stemsTap = tap; _mode = 'stems'; - console.log('[renderer-bus] engaged: stems graph → engine bus'); + console.log('[renderer-bus] engaged: stems graph → ' + + (viaChannel ? ('mixer channel #' + _stemsChannelId) : 'engine bus (legacy)')); } async function _engageElement() {