From 3d344695353f91d694902f2d51e6bc559835c90c Mon Sep 17 00:00:00 2001 From: OmikronApex Date: Sat, 11 Jul 2026 13:14:05 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(audio):=20loopback=20feeder=20mode=20?= =?UTF-8?q?=E2=80=94=20all=20app=20audio=20under=20exclusive/ASIO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tester-confirmed (2026-07-11 log): song previews and other plugin-private audio bypass the per-surface feeder taps and leak to the default WASAPI device under ASIO output. Also confirmed: the element capture path poisons itself when highway_3d already owns #audio's one-shot MediaElementSource (InvalidStateError with _elCtx assigned pre-throw → TypeError every later tick). - New preferred mode 'loopback': one getDisplayMedia frame-audio capture (desktop main answers with the app's own frame) covers song, previews, and UI sounds for the whole exclusive session — engages even with no song loaded. Local playback silenced via suppressLocalAudioPlayback, page-mute IPC fallback otherwise. - Sticky fallback to the existing stems/element surface modes when capture is unavailable (old desktop main, denied, Docker sphere). - Element capture: assign module state only after the whole chain succeeds; close the context on failure — collision now retries clean. - Failed engage now disables the bus and tears down loopback (no more bus-enabled-with-no-producer stranding). - Tests: 12 (5 new — loopback engage/preference/mute-fallback/sticky fallback, collision retry). Co-Authored-By: Claude Fable 5 --- static/app.js | 121 +++++++++++++++++++++-- tests/js/renderer_bus_feeder.test.js | 140 ++++++++++++++++++++++++--- 2 files changed, 240 insertions(+), 21 deletions(-) diff --git a/static/app.js b/static/app.js index d5d33ef..48a812b 100644 --- a/static/app.js +++ b/static/app.js @@ -5375,15 +5375,85 @@ window.jucePlayer = jucePlayer; if (_elCtx) return; const el = document.getElementById('audio'); if (!el) throw new Error('no core audio element'); - _elCtx = new AudioContext(); - _elSource = _elCtx.createMediaElementSource(el); - _elSource.connect(_elCtx.destination); - _elTap = _makeTap(_elCtx); - await _elTap.attach(_elSource); + // Assign the module state ONLY after the whole chain succeeded. + // createMediaElementSource throws InvalidStateError when another + // consumer (highway_3d's analyser tap) already owns the element's + // one-shot source — assigning _elCtx before that throw poisoned every + // later tick into `_elTap.active` TypeErrors (tester log 2026-07-11) + // while the song kept playing on the default device. + const ctx = new AudioContext(); + let source, tap; + try { + source = ctx.createMediaElementSource(el); + source.connect(ctx.destination); + tap = _makeTap(ctx); + await tap.attach(source); + } catch (e) { + try { await ctx.close(); } catch (_) { /* already closed */ } + throw e; + } + _elCtx = ctx; _elSource = source; _elTap = tap; + } + + // ── Whole-app loopback capture ─────────────────────────────────────────── + // Preferred mode: one getDisplayMedia frame-audio capture covers EVERY + // sound the app makes (song, previews, UI) — no per-surface taps, so + // plugin-private AudioContexts (song-preview, future plugins) survive + // exclusive/ASIO output too. The desktop main process answers the request + // with this window's own frame (frame-scoped — no other apps' audio). + // Local playback is silenced via the suppressLocalAudioPlayback track + // constraint, with a page-mute IPC fallback (capture taps frame audio + // before the output mute, so a muted page still feeds the stream). + let _lbStream = null, _lbCtx = null, _lbTap = null, _lbPageMuted = false; + let _loopbackUnavailable = false; // sticky: probe once, then fall back + async function _engageLoopback() { + const stream = await navigator.mediaDevices.getDisplayMedia({ + video: true, + audio: { suppressLocalAudioPlayback: true }, + }); + for (const t of stream.getVideoTracks()) t.stop(); // required, unused + const track = stream.getAudioTracks()[0]; + if (!track) { + for (const t of stream.getTracks()) t.stop(); + throw new Error('no loopback audio track'); + } + try { + _lbCtx = _lbCtx || new AudioContext(); + if (_lbCtx.state !== 'running') await _lbCtx.resume().catch(() => {}); + const source = _lbCtx.createMediaStreamSource(stream); + const tap = _makeTap(_lbCtx); + await tap.attach(source); + const suppressed = track.getSettings?.().suppressLocalAudioPlayback === true; + if (!suppressed && typeof api.setPageMuted === 'function') { + _lbPageMuted = (await api.setPageMuted(true)) === true; + } + if (window._asioDiagEnabled?.()) { + console.log('[asio-diag] loopback: suppressed=', suppressed, + 'pageMuted=', _lbPageMuted, 'rate=', _lbCtx.sampleRate); + } + await api.setRendererBus(true, 1.0); + tap.active = true; + _lbStream = stream; _lbTap = tap; + _mode = 'loopback'; + console.log('[renderer-bus] engaged: app loopback → engine bus'); + } catch (e) { + for (const t of stream.getTracks()) t.stop(); + throw e; + } + } + async function _teardownLoopback() { + if (_lbTap) _lbTap.active = false; + if (_lbStream) for (const t of _lbStream.getTracks()) t.stop(); + _lbStream = null; _lbTap = null; + if (_lbPageMuted && typeof api.setPageMuted === 'function') { + try { await api.setPageMuted(false); } catch (_) { /* engine gone */ } + } + _lbPageMuted = false; } // ── Engagement state machine ───────────────────────────────────────────── - // 'off' | 'element' | 'stems' + // 'off' | 'loopback' | 'element' | 'stems' (element/stems = fallback when + // loopback capture is unavailable: old desktop main, denied capture) let _mode = 'off'; let _stemsGraph = null; // { context, masterNode } snapshot while engaged let _stemsTap = null; @@ -5408,7 +5478,9 @@ window.jucePlayer = jucePlayer; const prev = _mode; _mode = 'off'; try { await api.setRendererBus(false, 0); } catch (_) { /* engine gone */ } - if (prev === 'element' && _elCtx) { + if (prev === 'loopback') { + await _teardownLoopback(); + } else if (prev === 'element' && _elCtx) { _elTap.active = false; await _setSink(_elCtx, false).catch(() => {}); } else if (prev === 'stems' && _stemsGraph) { @@ -5467,9 +5539,20 @@ window.jucePlayer = jucePlayer; let want = 'off'; if (running && exclusive) { - if (stems) want = 'stems'; + // Loopback covers ALL app audio (song, previews, UI), so it + // engages for the whole exclusive session — not just while a + // song is loaded. Per-surface modes remain as fallback when + // loopback capture is unavailable (old desktop main without + // the display-media handler, capture denied). + if (!_loopbackUnavailable) want = 'loopback'; + else if (stems) want = 'stems'; else if (elementSong) want = 'element'; } + // Song audio riding the native transport must not ALSO ride the + // loopback (double-carry into the same engine output). The native + // transport plays from the engine, not the page, so page loopback + // never hears it — no conflict; loopback stays engaged for + // previews/UI while the transport owns the song. // [asio-diag] full decision vector, change-gated (500ms poll — // steady state must not flood the buffer). This is the feeder-side @@ -5481,6 +5564,7 @@ window.jucePlayer = jucePlayer; + ' stems=' + !!stems + ' songAudio=' + !!songAudio + ' juceMode=' + !!window._juceMode + ' elementSong=' + elementSong + + ' loopbackUnavailable=' + _loopbackUnavailable + ' want=' + want + ' mode=' + _mode; if (d !== window._lastRendererBusDecision) { window._lastRendererBusDecision = d; @@ -5492,12 +5576,29 @@ window.jucePlayer = jucePlayer; const stemsGraphChanged = _mode === 'stems' && stems !== _stemsGraph; if (want !== _mode || stemsGraphChanged) { await _disengage(); - if (want === 'stems') await _engageStems(stems); - else if (want === 'element') await _engageElement(); + try { + if (want === 'loopback') await _engageLoopback(); + else if (want === 'stems') await _engageStems(stems); + else if (want === 'element') await _engageElement(); + } catch (e) { + if (want === 'loopback') { + // Capture unavailable (no handler in an old desktop + // main, permission denied) — remember and fall back to + // the per-surface modes on the next tick. + _loopbackUnavailable = true; + console.warn('[renderer-bus] loopback capture unavailable — falling back to surface taps:', e); + } + throw e; + } } } catch (e) { console.warn('[renderer-bus] reevaluate failed (will retry):', e); _mode = 'off'; + // A partial engage may have left the bus enabled with no producer + // and the page muted — undo both so a failed tick can't strand + // audio in silence until the next successful engage. + try { await api.setRendererBus(false, 0); } catch (_) { /* engine gone */ } + await _teardownLoopback().catch(() => {}); } finally { _busy = false; } diff --git a/tests/js/renderer_bus_feeder.test.js b/tests/js/renderer_bus_feeder.test.js index 43535e6..8438db7 100644 --- a/tests/js/renderer_bus_feeder.test.js +++ b/tests/js/renderer_bus_feeder.test.js @@ -50,17 +50,41 @@ function makeFakeContext(sampleRate = 48000) { this.mediaSourceEl = el; return { connect() {}, disconnect() {} }; }, + createMediaStreamSource(stream) { + this.mediaStreamSource = stream; + return { connect() {}, disconnect() {} }; + }, }; return ctx; } -function makeSandbox({ isAudioRunning = () => true, exclusive = () => true } = {}) { - const calls = { setRendererBus: [], pushRendererAudio: [] }; +// Fake getDisplayMedia stream for the loopback-capture path. +function makeLoopbackStream({ suppressed = true } = {}) { + const stopped = []; + const audioTrack = { + kind: 'audio', + stop() { stopped.push('audio'); }, + getSettings: () => (suppressed ? { suppressLocalAudioPlayback: true } : {}), + }; + const videoTrack = { kind: 'video', stop() { stopped.push('video'); } }; + return { + __stopped: stopped, + getAudioTracks: () => [audioTrack], + getVideoTracks: () => [videoTrack], + getTracks: () => [videoTrack, audioTrack], + }; +} + +// `displayMedia`: undefined → loopback capture unavailable (Docker sphere / +// old desktop main); a function → used as navigator.mediaDevices.getDisplayMedia. +function makeSandbox({ isAudioRunning = () => true, exclusive = () => true, displayMedia } = {}) { + const calls = { setRendererBus: [], pushRendererAudio: [], setPageMuted: [] }; const api = { isAudioRunning: () => Promise.resolve(isAudioRunning()), setRendererBus: (en, g) => { calls.setRendererBus.push([en, g]); return Promise.resolve(); }, pushRendererAudio: (buf, rate) => { calls.pushRendererAudio.push([buf.length, rate]); }, + setPageMuted: (m) => { calls.setPageMuted.push(m); return Promise.resolve(m); }, }; class FakeWorkletNode { @@ -85,6 +109,7 @@ function makeSandbox({ isAudioRunning = () => true, exclusive = () => true } = { __createdContexts: [], __audioEl: { id: 'audio' }, __calls: calls, + navigator: { mediaDevices: displayMedia ? { getDisplayMedia: displayMedia } : {} }, window: null, }; sandbox.window = { @@ -111,12 +136,21 @@ function makeStemsGraph() { }; } -test('stems graph + exclusive output → bus enabled, stems ctx null-sinked', async () => { +// Surface-mode (stems/element) tests run WITHOUT getDisplayMedia: the first +// tick probes loopback, fails, and latches _loopbackUnavailable; the second +// tick exercises the fallback surface mode. This mirrors an old desktop main +// without the display-media handler. +async function reevaluateWithFallback(sb) { + await sb.window._reevaluateRendererBus(); // loopback probe → unavailable + await sb.window._reevaluateRendererBus(); // surface fallback +} + +test('stems graph + exclusive output → bus enabled, stems ctx null-sinked (loopback unavailable)', async () => { const sb = makeSandbox({ exclusive: () => true }); const graph = makeStemsGraph(); sb.window.feedBack.stems.audioGraph = graph; - await sb.window._reevaluateRendererBus(); + await reevaluateWithFallback(sb); assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled'); assert.equal(graph.context.sinkIdCalls.at(-1)?.type, 'none', 'stems ctx re-pointed at null sink'); @@ -128,7 +162,7 @@ test('output returns to shared → bus disabled, sink restored', async () => { const graph = makeStemsGraph(); sb.window.feedBack.stems.audioGraph = graph; - await sb.window._reevaluateRendererBus(); + await reevaluateWithFallback(sb); excl = false; await sb.window._reevaluateRendererBus(); @@ -145,26 +179,27 @@ test('stems graph + shared output → feeder stays off (no double audio)', async assert.equal(sb.__calls.setRendererBus.length, 0, 'bus never touched in shared mode'); }); -test('element song + exclusive → element captured into bus', async () => { +test('element song + exclusive → element captured into bus (loopback unavailable)', async () => { const sb = makeSandbox({ exclusive: () => true }); sb.window._currentSongAudio = { url: '/api/sloppak/x.sloppak/file/stems/full.ogg' }; sb.window._juceMode = false; - await sb.window._reevaluateRendererBus(); + await reevaluateWithFallback(sb); assert.equal(sb.__createdContexts.length, 1, 'capture context created'); assert.equal(sb.__createdContexts[0].mediaSourceEl, sb.__audioEl, 'element source captured'); assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled'); }); -test('song riding the native transport (_juceMode) → feeder stays off', async () => { +test('native-transport song, loopback unavailable → surface modes stay off', async () => { const sb = makeSandbox({ exclusive: () => true }); sb.window._currentSongAudio = { url: '/audio/song.ogg' }; sb.window._juceMode = true; - await sb.window._reevaluateRendererBus(); + await reevaluateWithFallback(sb); - assert.equal(sb.__calls.setRendererBus.length, 0, 'native transport owns the song'); + assert.ok(!sb.__calls.setRendererBus.some(([en]) => en === true), + 'bus never ENABLED (failed-probe cleanup may disable it)'); assert.equal(sb.__createdContexts.length, 0, 'no capture context created'); }); @@ -172,7 +207,7 @@ test('stems graph replaced mid-engagement → re-engages on the new graph', asyn const sb = makeSandbox({ exclusive: () => true }); const g1 = makeStemsGraph(); sb.window.feedBack.stems.audioGraph = g1; - await sb.window._reevaluateRendererBus(); + await reevaluateWithFallback(sb); const g2 = makeStemsGraph(); sb.window.feedBack.stems.audioGraph = g2; @@ -182,6 +217,89 @@ test('stems graph replaced mid-engagement → re-engages on the new graph', asyn assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 're-enabled for new graph'); }); +// ── Loopback mode (whole-app capture) ──────────────────────────────────────── + +test('exclusive output + loopback available → engages without any song loaded', async () => { + const stream = makeLoopbackStream(); + const sb = makeSandbox({ exclusive: () => true, displayMedia: () => Promise.resolve(stream) }); + + await sb.window._reevaluateRendererBus(); + + assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled for whole session'); + assert.ok(stream.__stopped.includes('video'), 'unused video track stopped'); + assert.equal(sb.__createdContexts.at(-1)?.mediaStreamSource, stream, 'loopback stream captured'); + assert.equal(sb.__calls.setPageMuted.length, 0, 'suppress constraint honoured — no page mute'); +}); + +test('loopback preferred over stems when both available', async () => { + const stream = makeLoopbackStream(); + const sb = makeSandbox({ exclusive: () => true, displayMedia: () => Promise.resolve(stream) }); + const graph = makeStemsGraph(); + sb.window.feedBack.stems.audioGraph = graph; + + await sb.window._reevaluateRendererBus(); + + assert.equal(graph.context.sinkIdCalls.length, 0, 'stems ctx untouched — loopback owns capture'); + assert.equal(sb.__createdContexts.at(-1)?.mediaStreamSource, stream, 'loopback engaged'); +}); + +test('suppressLocalAudioPlayback unsupported → page-mute fallback, unmuted on disengage', async () => { + let excl = true; + const stream = makeLoopbackStream({ suppressed: false }); + const sb = makeSandbox({ exclusive: () => excl, displayMedia: () => Promise.resolve(stream) }); + + await sb.window._reevaluateRendererBus(); + assert.deepEqual(sb.__calls.setPageMuted, [true], 'page muted as fallback'); + + excl = false; + await sb.window._reevaluateRendererBus(); + assert.deepEqual(sb.__calls.setPageMuted, [true, false], 'page unmuted on disengage'); + assert.deepEqual(sb.__calls.setRendererBus.at(-1), [false, 0], 'bus disabled'); +}); + +test('getDisplayMedia rejected → sticky fallback to surface modes', async () => { + const sb = makeSandbox({ + exclusive: () => true, + displayMedia: () => Promise.reject(new DOMException('denied', 'NotAllowedError')), + }); + const graph = makeStemsGraph(); + sb.window.feedBack.stems.audioGraph = graph; + + await sb.window._reevaluateRendererBus(); // probe fails, latches unavailable + await sb.window._reevaluateRendererBus(); // falls back to stems + + assert.equal(graph.context.sinkIdCalls.at(-1)?.type, 'none', 'stems fallback engaged'); + assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled via fallback'); +}); + +test('element capture collision (createMediaElementSource throws) → no poisoned state, clean retry', async () => { + const sb = makeSandbox({ exclusive: () => true }); // loopback unavailable + sb.window._currentSongAudio = { url: '/api/sloppak/x.sloppak/file/stems/full.ogg' }; + // First capture attempt collides (highway analyser owns the element). + let collide = true; + const origFactory = sb.AudioContext; + sb.__createdContexts.length = 0; + // Patch contexts so createMediaElementSource throws while colliding. + sb.AudioContext = function () { + const c = origFactory(); + const orig = c.createMediaElementSource.bind(c); + c.createMediaElementSource = (el) => { + if (collide) throw new DOMException('already connected', 'InvalidStateError'); + return orig(el); + }; + c.close = () => Promise.resolve(); + return c; + }; + + await reevaluateWithFallback(sb); // element engage fails (collision) + assert.ok(!sb.__calls.setRendererBus.some(([en]) => en === true), 'bus never left enabled'); + + collide = false; + await sb.window._reevaluateRendererBus(); // retry succeeds — no TypeError, fresh ctx + + assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'element engaged after collision cleared'); +}); + test('engine stops → bus disabled', async () => { let running = true; const sb = makeSandbox({ isAudioRunning: () => running, exclusive: () => true }); From 8f53ea11ae00b66fed3539078558965a85c79b3d Mon Sep 17 00:00:00 2001 From: Byron Gamatos Date: Sat, 11 Jul 2026 13:41:33 +0200 Subject: [PATCH 2/2] fix(audio): close loopback capture context on teardown (release tap worklet) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loopback context was reused across engages (_lbCtx || new), but teardown only stopped the stream + deactivated the tap — never closing the context or detaching the worklet node. Each exclusive<->shared switch orphaned a live tap worklet on the long-lived context. Use a fresh context per session and close it on disengage. Adds a test asserting the context is closed on teardown. Co-Authored-By: Claude Opus 4.8 (1M context) --- static/app.js | 11 ++++++++++- tests/js/renderer_bus_feeder.test.js | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/static/app.js b/static/app.js index 48a812b..ce33c97 100644 --- a/static/app.js +++ b/static/app.js @@ -5418,7 +5418,9 @@ window.jucePlayer = jucePlayer; throw new Error('no loopback audio track'); } try { - _lbCtx = _lbCtx || new AudioContext(); + // Fresh context per session (not reused) so teardown's close() + // fully releases the tap worklet node — see _teardownLoopback. + _lbCtx = new AudioContext(); if (_lbCtx.state !== 'running') await _lbCtx.resume().catch(() => {}); const source = _lbCtx.createMediaStreamSource(stream); const tap = _makeTap(_lbCtx); @@ -5445,6 +5447,13 @@ window.jucePlayer = jucePlayer; if (_lbTap) _lbTap.active = false; if (_lbStream) for (const t of _lbStream.getTracks()) t.stop(); _lbStream = null; _lbTap = null; + // Close the capture context so its tap worklet node is released. The + // context is per-session (not reused): without this, each exclusive⇄ + // shared switch orphaned a live worklet on a long-lived context. + if (_lbCtx) { + try { await _lbCtx.close(); } catch (_) { /* already closed */ } + _lbCtx = null; + } if (_lbPageMuted && typeof api.setPageMuted === 'function') { try { await api.setPageMuted(false); } catch (_) { /* engine gone */ } } diff --git a/tests/js/renderer_bus_feeder.test.js b/tests/js/renderer_bus_feeder.test.js index 8438db7..1500816 100644 --- a/tests/js/renderer_bus_feeder.test.js +++ b/tests/js/renderer_bus_feeder.test.js @@ -54,6 +54,7 @@ function makeFakeContext(sampleRate = 48000) { this.mediaStreamSource = stream; return { connect() {}, disconnect() {} }; }, + close() { this.closed = true; return Promise.resolve(); }, }; return ctx; } @@ -231,6 +232,22 @@ test('exclusive output + loopback available → engages without any song loaded' assert.equal(sb.__calls.setPageMuted.length, 0, 'suppress constraint honoured — no page mute'); }); +test('loopback context is closed on disengage (no orphaned tap worklet)', async () => { + let excl = true; + const stream = makeLoopbackStream(); + const sb = makeSandbox({ exclusive: () => excl, displayMedia: () => Promise.resolve(stream) }); + + await sb.window._reevaluateRendererBus(); // engage loopback + const lbCtx = sb.__createdContexts.at(-1); + assert.equal(lbCtx?.mediaStreamSource, stream, 'loopback engaged'); + assert.notEqual(lbCtx.closed, true, 'context live while engaged'); + + excl = false; + await sb.window._reevaluateRendererBus(); // disengage + assert.equal(lbCtx.closed, true, 'loopback context closed on disengage'); + assert.ok(stream.__stopped.includes('audio'), 'capture stream stopped'); +}); + test('loopback preferred over stems when both available', async () => { const stream = makeLoopbackStream(); const sb = makeSandbox({ exclusive: () => true, displayMedia: () => Promise.resolve(stream) });