feat(audio): loopback feeder mode — all app audio under exclusive/ASIO (#865)

* feat(audio): loopback feeder mode — all app audio under exclusive/ASIO Tester-confirmed (2026-07-11 log): song previews and other plugin-private audio bypass the per-surface feeder taps and leak to the default WASAPI device under ASIO output. Also confirmed: the element capture path poisons itself when highway_3d already owns #audio's one-shot MediaElementSource (InvalidStateError with _elCtx assigned pre-throw → TypeError every later tick). - New preferred mode 'loopback': one getDisplayMedia frame-audio capture (desktop main answers with the app's own frame) covers song, previews, and UI sounds for the whole exclusive session — engages even with no song loaded. Local playback silenced via suppressLocalAudioPlayback, page-mute IPC fallback otherwise. - Sticky fallback to the existing stems/element surface modes when capture is unavailable (old desktop main, denied, Docker sphere). - Element capture: assign module state only after the whole chain succeeds; close the context on failure — collision now retries clean. - Failed engage now disables the bus and tears down loopback (no more bus-enabled-with-no-producer stranding). - Tests: 12 (5 new — loopback engage/preference/mute-fallback/sticky fallback, collision retry). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(audio): close loopback capture context on teardown (release tap worklet) The loopback context was reused across engages (_lbCtx || new), but teardown only stopped the stream + deactivated the tap — never closing the context or detaching the worklet node. Each exclusive<->shared switch orphaned a live tap worklet on the long-lived context. Use a fresh context per session and close it on disengage. Adds a test asserting the context is closed on teardown. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Byron Gamatos <xasiklas@gmail.com>
2026-08-11 03:09:57 +00:00 · 2026-07-11 14:54:40 +02:00
parent bbdff4e10f
commit 9a58a55fe8
2 changed files with 266 additions and 21 deletions
@@ -50,17 +50,42 @@ function makeFakeContext(sampleRate = 48000) {
            this.mediaSourceEl = el;
            return { connect() {}, disconnect() {} };
        },
+        createMediaStreamSource(stream) {
+            this.mediaStreamSource = stream;
+            return { connect() {}, disconnect() {} };
+        },
+        close() { this.closed = true; return Promise.resolve(); },
    };
    return ctx;
 }

-function makeSandbox({ isAudioRunning = () => true, exclusive = () => true } = {}) {
-    const calls = { setRendererBus: [], pushRendererAudio: [] };
+// Fake getDisplayMedia stream for the loopback-capture path.
+function makeLoopbackStream({ suppressed = true } = {}) {
+    const stopped = [];
+    const audioTrack = {
+        kind: 'audio',
+        stop() { stopped.push('audio'); },
+        getSettings: () => (suppressed ? { suppressLocalAudioPlayback: true } : {}),
+    };
+    const videoTrack = { kind: 'video', stop() { stopped.push('video'); } };
+    return {
+        __stopped: stopped,
+        getAudioTracks: () => [audioTrack],
+        getVideoTracks: () => [videoTrack],
+        getTracks: () => [videoTrack, audioTrack],
+    };
+}
+
+// `displayMedia`: undefined → loopback capture unavailable (Docker sphere /
+// old desktop main); a function → used as navigator.mediaDevices.getDisplayMedia.
+function makeSandbox({ isAudioRunning = () => true, exclusive = () => true, displayMedia } = {}) {
+    const calls = { setRendererBus: [], pushRendererAudio: [], setPageMuted: [] };

    const api = {
        isAudioRunning: () => Promise.resolve(isAudioRunning()),
        setRendererBus: (en, g) => { calls.setRendererBus.push([en, g]); return Promise.resolve(); },
        pushRendererAudio: (buf, rate) => { calls.pushRendererAudio.push([buf.length, rate]); },
+        setPageMuted: (m) => { calls.setPageMuted.push(m); return Promise.resolve(m); },
    };

    class FakeWorkletNode {
@@ -85,6 +110,7 @@ function makeSandbox({ isAudioRunning = () => true, exclusive = () => true } = {
        __createdContexts: [],
        __audioEl: { id: 'audio' },
        __calls: calls,
+        navigator: { mediaDevices: displayMedia ? { getDisplayMedia: displayMedia } : {} },
        window: null,
    };
    sandbox.window = {
@@ -111,12 +137,21 @@ function makeStemsGraph() {
    };
 }

-test('stems graph + exclusive output → bus enabled, stems ctx null-sinked', async () => {
+// Surface-mode (stems/element) tests run WITHOUT getDisplayMedia: the first
+// tick probes loopback, fails, and latches _loopbackUnavailable; the second
+// tick exercises the fallback surface mode. This mirrors an old desktop main
+// without the display-media handler.
+async function reevaluateWithFallback(sb) {
+    await sb.window._reevaluateRendererBus();   // loopback probe → unavailable
+    await sb.window._reevaluateRendererBus();   // surface fallback
+}
+
+test('stems graph + exclusive output → bus enabled, stems ctx null-sinked (loopback unavailable)', async () => {
    const sb = makeSandbox({ exclusive: () => true });
    const graph = makeStemsGraph();
    sb.window.feedBack.stems.audioGraph = graph;

-    await sb.window._reevaluateRendererBus();
+    await reevaluateWithFallback(sb);

    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled');
    assert.equal(graph.context.sinkIdCalls.at(-1)?.type, 'none', 'stems ctx re-pointed at null sink');
@@ -128,7 +163,7 @@ test('output returns to shared → bus disabled, sink restored', async () => {
    const graph = makeStemsGraph();
    sb.window.feedBack.stems.audioGraph = graph;

-    await sb.window._reevaluateRendererBus();
+    await reevaluateWithFallback(sb);
    excl = false;
    await sb.window._reevaluateRendererBus();

@@ -145,26 +180,27 @@ test('stems graph + shared output → feeder stays off (no double audio)', async
    assert.equal(sb.__calls.setRendererBus.length, 0, 'bus never touched in shared mode');
 });

-test('element song + exclusive → element captured into bus', async () => {
+test('element song + exclusive → element captured into bus (loopback unavailable)', async () => {
    const sb = makeSandbox({ exclusive: () => true });
    sb.window._currentSongAudio = { url: '/api/sloppak/x.sloppak/file/stems/full.ogg' };
    sb.window._juceMode = false;

-    await sb.window._reevaluateRendererBus();
+    await reevaluateWithFallback(sb);

    assert.equal(sb.__createdContexts.length, 1, 'capture context created');
    assert.equal(sb.__createdContexts[0].mediaSourceEl, sb.__audioEl, 'element source captured');
    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled');
 });

-test('song riding the native transport (_juceMode) → feeder stays off', async () => {
+test('native-transport song, loopback unavailable → surface modes stay off', async () => {
    const sb = makeSandbox({ exclusive: () => true });
    sb.window._currentSongAudio = { url: '/audio/song.ogg' };
    sb.window._juceMode = true;

-    await sb.window._reevaluateRendererBus();
+    await reevaluateWithFallback(sb);

-    assert.equal(sb.__calls.setRendererBus.length, 0, 'native transport owns the song');
+    assert.ok(!sb.__calls.setRendererBus.some(([en]) => en === true),
+        'bus never ENABLED (failed-probe cleanup may disable it)');
    assert.equal(sb.__createdContexts.length, 0, 'no capture context created');
 });

@@ -172,7 +208,7 @@ test('stems graph replaced mid-engagement → re-engages on the new graph', asyn
    const sb = makeSandbox({ exclusive: () => true });
    const g1 = makeStemsGraph();
    sb.window.feedBack.stems.audioGraph = g1;
-    await sb.window._reevaluateRendererBus();
+    await reevaluateWithFallback(sb);

    const g2 = makeStemsGraph();
    sb.window.feedBack.stems.audioGraph = g2;
@@ -182,6 +218,105 @@ test('stems graph replaced mid-engagement → re-engages on the new graph', asyn
    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 're-enabled for new graph');
 });

+// ── Loopback mode (whole-app capture) ────────────────────────────────────────
+
+test('exclusive output + loopback available → engages without any song loaded', async () => {
+    const stream = makeLoopbackStream();
+    const sb = makeSandbox({ exclusive: () => true, displayMedia: () => Promise.resolve(stream) });
+
+    await sb.window._reevaluateRendererBus();
+
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled for whole session');
+    assert.ok(stream.__stopped.includes('video'), 'unused video track stopped');
+    assert.equal(sb.__createdContexts.at(-1)?.mediaStreamSource, stream, 'loopback stream captured');
+    assert.equal(sb.__calls.setPageMuted.length, 0, 'suppress constraint honoured — no page mute');
+});
+
+test('loopback context is closed on disengage (no orphaned tap worklet)', async () => {
+    let excl = true;
+    const stream = makeLoopbackStream();
+    const sb = makeSandbox({ exclusive: () => excl, displayMedia: () => Promise.resolve(stream) });
+
+    await sb.window._reevaluateRendererBus();          // engage loopback
+    const lbCtx = sb.__createdContexts.at(-1);
+    assert.equal(lbCtx?.mediaStreamSource, stream, 'loopback engaged');
+    assert.notEqual(lbCtx.closed, true, 'context live while engaged');
+
+    excl = false;
+    await sb.window._reevaluateRendererBus();          // disengage
+    assert.equal(lbCtx.closed, true, 'loopback context closed on disengage');
+    assert.ok(stream.__stopped.includes('audio'), 'capture stream stopped');
+});
+
+test('loopback preferred over stems when both available', async () => {
+    const stream = makeLoopbackStream();
+    const sb = makeSandbox({ exclusive: () => true, displayMedia: () => Promise.resolve(stream) });
+    const graph = makeStemsGraph();
+    sb.window.feedBack.stems.audioGraph = graph;
+
+    await sb.window._reevaluateRendererBus();
+
+    assert.equal(graph.context.sinkIdCalls.length, 0, 'stems ctx untouched — loopback owns capture');
+    assert.equal(sb.__createdContexts.at(-1)?.mediaStreamSource, stream, 'loopback engaged');
+});
+
+test('suppressLocalAudioPlayback unsupported → page-mute fallback, unmuted on disengage', async () => {
+    let excl = true;
+    const stream = makeLoopbackStream({ suppressed: false });
+    const sb = makeSandbox({ exclusive: () => excl, displayMedia: () => Promise.resolve(stream) });
+
+    await sb.window._reevaluateRendererBus();
+    assert.deepEqual(sb.__calls.setPageMuted, [true], 'page muted as fallback');
+
+    excl = false;
+    await sb.window._reevaluateRendererBus();
+    assert.deepEqual(sb.__calls.setPageMuted, [true, false], 'page unmuted on disengage');
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [false, 0], 'bus disabled');
+});
+
+test('getDisplayMedia rejected → sticky fallback to surface modes', async () => {
+    const sb = makeSandbox({
+        exclusive: () => true,
+        displayMedia: () => Promise.reject(new DOMException('denied', 'NotAllowedError')),
+    });
+    const graph = makeStemsGraph();
+    sb.window.feedBack.stems.audioGraph = graph;
+
+    await sb.window._reevaluateRendererBus();   // probe fails, latches unavailable
+    await sb.window._reevaluateRendererBus();   // falls back to stems
+
+    assert.equal(graph.context.sinkIdCalls.at(-1)?.type, 'none', 'stems fallback engaged');
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'bus enabled via fallback');
+});
+
+test('element capture collision (createMediaElementSource throws) → no poisoned state, clean retry', async () => {
+    const sb = makeSandbox({ exclusive: () => true });   // loopback unavailable
+    sb.window._currentSongAudio = { url: '/api/sloppak/x.sloppak/file/stems/full.ogg' };
+    // First capture attempt collides (highway analyser owns the element).
+    let collide = true;
+    const origFactory = sb.AudioContext;
+    sb.__createdContexts.length = 0;
+    // Patch contexts so createMediaElementSource throws while colliding.
+    sb.AudioContext = function () {
+        const c = origFactory();
+        const orig = c.createMediaElementSource.bind(c);
+        c.createMediaElementSource = (el) => {
+            if (collide) throw new DOMException('already connected', 'InvalidStateError');
+            return orig(el);
+        };
+        c.close = () => Promise.resolve();
+        return c;
+    };
+
+    await reevaluateWithFallback(sb);            // element engage fails (collision)
+    assert.ok(!sb.__calls.setRendererBus.some(([en]) => en === true), 'bus never left enabled');
+
+    collide = false;
+    await sb.window._reevaluateRendererBus();    // retry succeeds — no TypeError, fresh ctx
+
+    assert.deepEqual(sb.__calls.setRendererBus.at(-1), [true, 1.0], 'element engaged after collision cleared');
+});
+
 test('engine stops → bus disabled', async () => {
    let running = true;
    const sb = makeSandbox({ isAudioRunning: () => running, exclusive: () => true });