From 1c1a0e02681d4f3cf12f46f8eb903030f2339b74 Mon Sep 17 00:00:00 2001
From: OmikronApex <45161725+OmikronApex@users.noreply.github.com>
Date: Thu, 9 Jul 2026 22:51:09 +0200
Subject: [PATCH] =?UTF-8?q?feat(audio):=20renderer-bus=20feeder=20?=
 =?UTF-8?q?=E2=80=94=20song=20audio=20into=20engine=20output=20under=20exc?=
 =?UTF-8?q?lusive=20mode=20(Phase=202)=20(#828)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat(audio): route feedpak full-mix natively under exclusive output

Song playback runs through the renderer, which WASAPI-exclusive (and
ASIO) output silences. Route single-mix feedpaks (stem-less
original_audio packs AND single-stem packs) onto the engine's backing
transport when the output device type is exclusive-style, and migrate
back to HTML5 when it isn't. Extends /api/audio-local-path to resolve
/api/sloppak/.../file/... URLs via the same containment guards as
serve_sloppak_file. Multi-stem packs stay on the WebAudio path
(Phase 2). Includes [feedpak-route] transition-gated diagnostics
logging.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(audio): renderer-bus feeder — mix renderer song audio into engine output (Phase 2)

Under exclusive-style output the native backing transport (Phase 1, #824)
carries loose /audio/ songs and feedpak full-mixes, but not the stems
plugin's multi-stem WebAudio graph or tracks JUCE rejected. The feeder taps
the renderer-side master with an AudioWorklet, re-points the owning
AudioContext at a null sink so it keeps rendering without a device, and
pushes ~10 ms chunks over IPC into the desktop engine's renderer bus
(feedBack-desktop#90 follow-up). Inert in the Docker sphere and in shared
mode. Validated by the fix12 tester spike: null-sink rendering works,
clocks hold, no overflow.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
---
 static/app.js                        | 203 +++++++++++++++++++++++++++
 tests/js/renderer_bus_feeder.test.js | 195 +++++++++++++++++++++++++
 2 files changed, 398 insertions(+)
 create mode 100644 tests/js/renderer_bus_feeder.test.js

diff --git a/static/app.js b/static/app.js
index e70eaad..3287bcb 100644
--- a/static/app.js
+++ b/static/app.js
@@ -5264,6 +5264,209 @@ window.jucePlayer = jucePlayer;
     }, 350);
 })();
 
+// Renderer-audio bus feeder (desktop Phase 2): when the engine holds the
+// output endpoint in an exclusive-style mode, Chromium cannot reach the
+// device, so any song audio still played by the renderer goes silent. The
+// Phase 1 watcher above already migrates what a single-file transport can
+// carry (loose /audio/ songs, feedpak full-mixes) onto the native backing
+// transport. This feeder covers the rest — the stems plugin's multi-stem
+// WebAudio graph, plus <audio>-element songs the native transport could not
+// take (e.g. a codec loadBackingTrack rejected).
+//
+// Mechanism: capture the renderer-side master with an AudioWorklet tap,
+// re-point the owning AudioContext at a null sink so it keeps rendering
+// without a device, and push ~10 ms chunks over IPC into the engine's
+// renderer bus, where they are mixed into the exclusive output like a
+// backing track (~10-20 ms added latency on song audio only; the guitar
+// monitoring path is untouched). Validated by the fix12 tester spike:
+// null-sink rendering works, clocks hold (drift → 0), no overflow.
+//
+// Docker sphere: window.feedBackDesktop is undefined → this whole block is
+// inert. Shared-mode desktop: the bus stays disabled (no double audio) and
+// captured contexts keep/regain their default sink.
+(function _installRendererBusFeeder() {
+    const api = window.feedBackDesktop?.audio;
+    if (!api || typeof api.setRendererBus !== 'function'
+             || typeof api.pushRendererAudio !== 'function') return;
+
+    const TAP_WORKLET = `
+        class FeedbackBusTap extends AudioWorkletProcessor {
+            process(inputs) {
+                const inp = inputs[0];
+                if (inp && inp[0]) {
+                    const L = inp[0], R = inp[1] || inp[0];
+                    const out = new Float32Array(L.length * 2);
+                    for (let i = 0; i < L.length; i++) { out[i*2] = L[i]; out[i*2+1] = R[i]; }
+                    this.port.postMessage(out, [out.buffer]);
+                }
+                return true;
+            }
+        }
+        registerProcessor('feedback-bus-tap', FeedbackBusTap);
+    `;
+    const _tapModuleUrl = URL.createObjectURL(new Blob([TAP_WORKLET], { type: 'application/javascript' }));
+    const _tapModuleLoaded = new WeakSet();   // AudioContexts with the module added
+
+    // One tap per captured graph. `active` gates the push (the worklet keeps
+    // running when inactive — it's silent bookkeeping, not audio).
+    function _makeTap(ctx) {
+        const state = { node: null, active: false, batch: [], batchFrames: 0 };
+        state.attach = async (sourceNode) => {
+            if (!_tapModuleLoaded.has(ctx)) {
+                await ctx.audioWorklet.addModule(_tapModuleUrl);
+                _tapModuleLoaded.add(ctx);
+            }
+            if (!state.node) {
+                state.node = new AudioWorkletNode(ctx, 'feedback-bus-tap', { numberOfInputs: 1, channelCount: 2 });
+                const BATCH = Math.round(ctx.sampleRate / 100);   // ~10 ms
+                state.node.port.onmessage = (e) => {
+                    if (!state.active) { state.batch = []; state.batchFrames = 0; return; }
+                    state.batch.push(e.data);
+                    state.batchFrames += e.data.length / 2;
+                    if (state.batchFrames >= BATCH) {
+                        const merged = new Float32Array(state.batchFrames * 2);
+                        let o = 0;
+                        for (const c of state.batch) { merged.set(c, o); o += c.length; }
+                        api.pushRendererAudio(merged, ctx.sampleRate);
+                        state.batch = []; state.batchFrames = 0;
+                    }
+                };
+            }
+            sourceNode.connect(state.node);
+            // No onward connection: the tap is a sink-side observer; audibility
+            // in shared mode comes from the graph's own destination path.
+        };
+        state.detach = (sourceNode) => {
+            state.active = false;
+            state.batch = []; state.batchFrames = 0;
+            if (state.node && sourceNode) {
+                try { sourceNode.disconnect(state.node); } catch (_) { /* already gone */ }
+            }
+        };
+        return state;
+    }
+
+    // ── Core <audio> element capture ─────────────────────────────────────────
+    // createMediaElementSource permanently reroutes the element into its
+    // context, so it is created lazily — only the first time an exclusive
+    // device actually needs it — and never torn down. From then on the element
+    // always plays through _elCtx; sink toggling routes it to the speakers
+    // (shared mode) or the null sink + bus (exclusive mode).
+    let _elCtx = null, _elSource = null, _elTap = null;
+    async function _ensureElementCapture() {
+        if (_elCtx) return;
+        const el = document.getElementById('audio');
+        if (!el) throw new Error('no core audio element');
+        _elCtx = new AudioContext();
+        _elSource = _elCtx.createMediaElementSource(el);
+        _elSource.connect(_elCtx.destination);
+        _elTap = _makeTap(_elCtx);
+        await _elTap.attach(_elSource);
+    }
+
+    // ── Engagement state machine ─────────────────────────────────────────────
+    // 'off' | 'element' | 'stems'
+    let _mode = 'off';
+    let _stemsGraph = null;   // { context, masterNode } snapshot while engaged
+    let _stemsTap = null;
+    const _stemsTaps = new WeakMap();  // context → tap (stems ctx is reused across songs)
+    let _busy = false;
+
+    async function _setSink(ctx, exclusive) {
+        if (typeof ctx.setSinkId !== 'function') throw new Error('setSinkId unsupported');
+        await ctx.setSinkId(exclusive ? { type: 'none' } : '');
+        if (ctx.state !== 'running') await ctx.resume().catch(() => {});
+    }
+
+    async function _disengage() {
+        if (_mode === 'off') return;
+        const prev = _mode;
+        _mode = 'off';
+        try { await api.setRendererBus(false, 0); } catch (_) { /* engine gone */ }
+        if (prev === 'element' && _elCtx) {
+            _elTap.active = false;
+            await _setSink(_elCtx, false).catch(() => {});
+        } else if (prev === 'stems' && _stemsGraph) {
+            if (_stemsTap) _stemsTap.detach(_stemsGraph.masterNode);
+            await _setSink(_stemsGraph.context, false).catch(() => {});
+            _stemsGraph = null; _stemsTap = null;
+        }
+        console.log('[renderer-bus] disengaged (' + prev + ')');
+    }
+
+    async function _engageStems(graph) {
+        await _setSink(graph.context, true);
+        let tap = _stemsTaps.get(graph.context);
+        if (!tap) { tap = _makeTap(graph.context); _stemsTaps.set(graph.context, tap); }
+        await tap.attach(graph.masterNode);
+        await api.setRendererBus(true, 1.0);
+        tap.active = true;
+        _stemsGraph = graph; _stemsTap = tap;
+        _mode = 'stems';
+        console.log('[renderer-bus] engaged: stems graph → engine bus');
+    }
+
+    async function _engageElement() {
+        await _ensureElementCapture();
+        await _setSink(_elCtx, true);
+        await api.setRendererBus(true, 1.0);
+        _elTap.active = true;
+        _mode = 'element';
+        console.log('[renderer-bus] engaged: <audio> element → engine bus');
+    }
+
+    async function _reevaluate() {
+        if (_busy) return;
+        _busy = true;
+        try {
+            let running = false, exclusive = false;
+            try {
+                running = await api.isAudioRunning();
+            } catch (_) { /* engine unreachable → treat as not running */ }
+            if (running) {
+                // Reuse the Phase 1 predicate installed by the routing watcher
+                // (getCurrentDevice + exclusive-type check with change-logged
+                // diagnostics). Fail closed if it is somehow absent.
+                exclusive = !!(await window._juceOutputIsExclusive?.());
+            }
+
+            // The stems plugin publishes its live graph while a multi-stem
+            // song is loaded (and removes it on teardown).
+            const stems = (window.feedBack || window.slopsmith)?.stems?.audioGraph || null;
+            // Element songs: a song is loaded, it is NOT riding the native
+            // transport (Phase 1 owns those), and the stems graph is not the
+            // player. Covers native-transport rejects (codec) in exclusive
+            // mode — without this they would be silent.
+            const songAudio = window._currentSongAudio;
+            const elementSong = !!songAudio && !window._juceMode && !stems;
+
+            let want = 'off';
+            if (running && exclusive) {
+                if (stems) want = 'stems';
+                else if (elementSong) want = 'element';
+            }
+
+            const stemsGraphChanged = _mode === 'stems' && stems !== _stemsGraph;
+            if (want !== _mode || stemsGraphChanged) {
+                await _disengage();
+                if (want === 'stems') await _engageStems(stems);
+                else if (want === 'element') await _engageElement();
+            }
+        } catch (e) {
+            console.warn('[renderer-bus] reevaluate failed (will retry):', e);
+            _mode = 'off';
+        } finally {
+            _busy = false;
+        }
+    }
+
+    // Same cadence/rationale as the routing watcher above. Also re-check on
+    // visibility return so a device switch made while hidden is reconciled.
+    setInterval(() => { if (!document.hidden) void _reevaluate(); }, 500);
+    document.addEventListener('visibilitychange', () => { if (!document.hidden) void _reevaluate(); });
+    window._reevaluateRendererBus = _reevaluate;
+})();
+
 // Desktop JUCE backing uses an empty <audio> element; plugins such as Section Map
 // still seek via audio.currentTime / pause / play. Mirror those onto jucePlayer
 // while _juceMode is active. Same-tick pause+seek coalesce into a single seek
diff --git a/tests/js/renderer_bus_feeder.test.js b/tests/js/renderer_bus_feeder.test.js
new file mode 100644
index 0000000..43535e6
--- /dev/null
+++ b/tests/js/renderer_bus_feeder.test.js
@@ -0,0 +1,195 @@
+// Behavioral tests for the renderer-audio bus feeder in static/app.js.
+//
+// The feeder (an IIFE, `_installRendererBusFeeder`) captures renderer-side
+// song audio (stems-plugin WebAudio master, or the core <audio> element) and
+// pushes it into the desktop engine's renderer bus while the output device is
+// exclusive-style — the Phase 2 path for audio the native backing transport
+// cannot carry. These tests extract that IIFE from source and exercise
+// `window._reevaluateRendererBus` against fakes, covering: stems engagement
+// under exclusive output, disengagement on return to shared mode, inertness
+// in shared mode / while the native transport owns the song, and the
+// element-capture fallback.
+
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const vm = require('node:vm');
+
+const APP_JS = path.join(__dirname, '..', '..', 'static', 'app.js');
+
+function extractFeederIIFE(src) {
+    const marker = '(function _installRendererBusFeeder() {';
+    const start = src.indexOf(marker);
+    assert.ok(start !== -1, 'feeder IIFE not found in app.js');
+    const openBrace = src.indexOf('{', start);
+    let depth = 1;
+    let i = openBrace + 1;
+    while (i < src.length && depth > 0) {
+        const ch = src[i];
+        if (ch === '{') depth++;
+        else if (ch === '}') depth--;
+        i++;
+    }
+    assert.ok(depth === 0, 'unbalanced braces in feeder IIFE');
+    const tail = src.slice(i, i + 5);
+    assert.match(tail, /^\)\(\)/, 'feeder IIFE not immediately invoked');
+    return src.slice(start, i) + ')();';
+}
+
+function makeFakeContext(sampleRate = 48000) {
+    const ctx = {
+        sampleRate,
+        state: 'running',
+        sinkIdCalls: [],
+        destination: { isDestination: true },
+        setSinkId(v) { this.sinkIdCalls.push(v); return Promise.resolve(); },
+        resume() { this.state = 'running'; return Promise.resolve(); },
+        audioWorklet: { addModule: () => Promise.resolve() },
+        createMediaElementSource(el) {
+            this.mediaSourceEl = el;
+            return { connect() {}, disconnect() {} };
+        },
+    };
+    return ctx;
+}
+
+function makeSandbox({ isAudioRunning = () => true, exclusive = () => true } = {}) {
+    const calls = { setRendererBus: [], pushRendererAudio: [] };
+
+    const api = {
+        isAudioRunning: () => Promise.resolve(isAudioRunning()),
+        setRendererBus: (en, g) => { calls.setRendererBus.push([en, g]); return Promise.resolve(); },
+        pushRendererAudio: (buf, rate) => { calls.pushRendererAudio.push([buf.length, rate]); },
+    };
+
+    class FakeWorkletNode {
+        constructor() { this.port = { onmessage: null }; }
+        connect() {}
+        disconnect() {}
+    }
+
+    const sandbox = {
+        console: { log() {}, warn() {}, error() {} },
+        URL: { createObjectURL: () => 'blob:tap', revokeObjectURL() {} },
+        Blob: class { constructor() {} },
+        AudioWorkletNode: FakeWorkletNode,
+        AudioContext: function () { const c = makeFakeContext(); sandbox.__createdContexts.push(c); return c; },
+        WeakSet, WeakMap, Promise, Float32Array, Math,
+        setInterval: () => 0,
+        document: {
+            hidden: false,
+            addEventListener() {},
+            getElementById: () => sandbox.__audioEl,
+        },
+        __createdContexts: [],
+        __audioEl: { id: 'audio' },
+        __calls: calls,
+        window: null,
+    };
+    sandbox.window = {
+        feedBackDesktop: { audio: api },
+        _juceOutputIsExclusive: () => Promise.resolve(exclusive()),
+        _juceMode: false,
+        _currentSongAudio: null,
+        feedBack: { stems: {} },
+    };
+    sandbox.globalThis = sandbox;
+
+    const src = fs.readFileSync(APP_JS, 'utf8');
+    vm.createContext(sandbox);
+    vm.runInContext(extractFeederIIFE(src), sandbox);
+    assert.equal(typeof sandbox.window._reevaluateRendererBus, 'function',
+        'feeder must expose window._reevaluateRendererBus');
+    return sandbox;
+}
+
+function makeStemsGraph() {
+    return {
+        context: makeFakeContext(),
+        masterNode: { connect() {}, disconnect() {} },
+    };
+}
+
+test('stems graph + exclusive output → bus enabled, stems ctx null-sinked', async () => {
+    const sb = makeSandbox({ exclusive: () => true });
+    const graph = makeStemsGraph();
+    sb.window.feedBack.stems.audioGraph = graph;
+
+    await sb.window._reevaluateRendererBus();
+
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled');
+    assert.equal(graph.context.sinkIdCalls.at(-1)?.type, 'none', 'stems ctx re-pointed at null sink');
+});
+
+test('output returns to shared → bus disabled, sink restored', async () => {
+    let excl = true;
+    const sb = makeSandbox({ exclusive: () => excl });
+    const graph = makeStemsGraph();
+    sb.window.feedBack.stems.audioGraph = graph;
+
+    await sb.window._reevaluateRendererBus();
+    excl = false;
+    await sb.window._reevaluateRendererBus();
+
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [false, 0], 'bus disabled');
+    assert.equal(graph.context.sinkIdCalls.at(-1), '', 'default sink restored');
+});
+
+test('stems graph + shared output → feeder stays off (no double audio)', async () => {
+    const sb = makeSandbox({ exclusive: () => false });
+    sb.window.feedBack.stems.audioGraph = makeStemsGraph();
+
+    await sb.window._reevaluateRendererBus();
+
+    assert.equal(sb.__calls.setRendererBus.length, 0, 'bus never touched in shared mode');
+});
+
+test('element song + exclusive → element captured into bus', async () => {
+    const sb = makeSandbox({ exclusive: () => true });
+    sb.window._currentSongAudio = { url: '/api/sloppak/x.sloppak/file/stems/full.ogg' };
+    sb.window._juceMode = false;
+
+    await sb.window._reevaluateRendererBus();
+
+    assert.equal(sb.__createdContexts.length, 1, 'capture context created');
+    assert.equal(sb.__createdContexts[0].mediaSourceEl, sb.__audioEl, 'element source captured');
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled');
+});
+
+test('song riding the native transport (_juceMode) → feeder stays off', async () => {
+    const sb = makeSandbox({ exclusive: () => true });
+    sb.window._currentSongAudio = { url: '/audio/song.ogg' };
+    sb.window._juceMode = true;
+
+    await sb.window._reevaluateRendererBus();
+
+    assert.equal(sb.__calls.setRendererBus.length, 0, 'native transport owns the song');
+    assert.equal(sb.__createdContexts.length, 0, 'no capture context created');
+});
+
+test('stems graph replaced mid-engagement → re-engages on the new graph', async () => {
+    const sb = makeSandbox({ exclusive: () => true });
+    const g1 = makeStemsGraph();
+    sb.window.feedBack.stems.audioGraph = g1;
+    await sb.window._reevaluateRendererBus();
+
+    const g2 = makeStemsGraph();
+    sb.window.feedBack.stems.audioGraph = g2;
+    await sb.window._reevaluateRendererBus();
+
+    assert.equal(g2.context.sinkIdCalls.at(-1)?.type, 'none', 'new graph null-sinked');
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 're-enabled for new graph');
+});
+
+test('engine stops → bus disabled', async () => {
+    let running = true;
+    const sb = makeSandbox({ isAudioRunning: () => running, exclusive: () => true });
+    sb.window.feedBack.stems.audioGraph = makeStemsGraph();
+    await sb.window._reevaluateRendererBus();
+
+    running = false;
+    await sb.window._reevaluateRendererBus();
+
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [false, 0], 'bus disabled after engine stop');
+});