From e78072448becc7e4c1aec6b6daa2db9e8c4804f9 Mon Sep 17 00:00:00 2001 From: Matthew Harris Glover Date: Fri, 17 Jul 2026 07:28:36 -0400 Subject: [PATCH 1/3] fix(build): force amd64 for the Linux Docker build on Apple Silicon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without --platform linux/amd64 on `docker build`, an Apple Silicon host produces a native arm64 image. bundle-python.sh always downloads the x86_64 python-build-standalone (the Linux target is x86_64-only end to end: Steam Deck, vgmstream, onnxruntime), so Rosetta ends up trying to exec a foreign-arch binary directly inside an otherwise-native container and fails (exit 133). Forcing amd64 on both build and run keeps the whole container consistently emulated. Also add SLOPSMITH_REPO (forwarded through the container, consumed by clone_slopsmith) so a fork branch — not just a fork ref on the upstream core repo — can be bundled for a local test build, matching the existing SLOPSMITH_REF override. --- scripts/build-common.sh | 9 +++++++-- scripts/build-linux-docker.sh | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/build-common.sh b/scripts/build-common.sh index fe2c303..be1dac3 100644 --- a/scripts/build-common.sh +++ b/scripts/build-common.sh @@ -107,10 +107,15 @@ clone_slopsmith() { # builds and the push/tag CI paths behave exactly as before. # --branch accepts either a branch or a tag, both shallow-cloneable. local slopsmith_ref="${SLOPSMITH_REF:-main}" + # SLOPSMITH_REPO overrides the core repo (default got-feedback/feedback), + # mirroring SLOPSMITH_REF. Lets a contributor without push access to the + # core repo bundle a branch pushed to their own fork for a test build: + # SLOPSMITH_REPO=me/feedBack SLOPSMITH_REF=my-branch + local slopsmith_repo="${SLOPSMITH_REPO:-got-feedback/feedback}" local _auth="" [[ -n "${GH_CLONE_TOKEN:-}" ]] && _auth="x-access-token:${GH_CLONE_TOKEN}@" - echo "Cloning Slopsmith repository (ref: ${slopsmith_ref})..." - git clone --depth 1 --branch "$slopsmith_ref" "https://${_auth}github.com/got-feedback/feedback.git" "$clone_dir" + echo "Cloning Slopsmith repository (${slopsmith_repo} ref: ${slopsmith_ref})..." + git clone --depth 1 --branch "$slopsmith_ref" "https://${_auth}github.com/${slopsmith_repo}.git" "$clone_dir" # Remove broken symlinks from plugins dir find "$clone_dir/plugins" -maxdepth 1 -type l -delete 2>/dev/null || true diff --git a/scripts/build-linux-docker.sh b/scripts/build-linux-docker.sh index a03249e..551130f 100755 --- a/scripts/build-linux-docker.sh +++ b/scripts/build-linux-docker.sh @@ -1,6 +1,15 @@ #!/bin/bash # Docker-based Linux build wrapper # Runs build-linux-ubuntu.sh inside a reproducible container +# +# --platform linux/amd64 is forced on both build and run: the Linux target is +# x86_64-only end to end (bundle-python.sh's python-build-standalone pin, +# vgmstream, onnxruntime, etc. have no arm64 Linux build, and Steam Deck +# itself is x86_64). On an Apple Silicon host, omitting --platform makes +# `docker build` produce a native arm64 image, so bundle-python.sh's hardcoded +# x86_64 download becomes a foreign-arch binary inside an otherwise-native +# container — Rosetta chokes trying to exec it directly instead of via full +# amd64 emulation. Forcing amd64 for the whole container sidesteps that. set -euo pipefail @@ -44,6 +53,7 @@ echo " (This will take a few minutes on first run)" echo "" docker build \ + --platform linux/amd64 \ -f "$DEVCONTAINER_DIR/Dockerfile" \ -t slopsmith-ubuntu-builder \ "$PROJECT_DIR" @@ -75,6 +85,7 @@ echo "" set +e docker run \ + --platform linux/amd64 \ --name "$CONTAINER_NAME" \ -v "$PROJECT_DIR:/workspace" \ -w /workspace \ @@ -83,6 +94,7 @@ docker run \ -e GIT_TERMINAL_PROMPT=0 \ -e "GH_CLONE_TOKEN=${GH_CLONE_TOKEN:-}" \ -e "SLOPSMITH_REF=${SLOPSMITH_REF:-main}" \ + -e "SLOPSMITH_REPO=${SLOPSMITH_REPO:-got-feedback/feedback}" \ -t \ slopsmith-ubuntu-builder \ bash -c './scripts/build-linux-ubuntu.sh' From 2dfdf02653320f588e9c1cdf05c6725fce1a0c69 Mon Sep 17 00:00:00 2001 From: Matthew Harris Glover Date: Fri, 17 Jul 2026 07:41:09 -0400 Subject: [PATCH 2/3] feat(build): fast local iteration path + ccache for the Linux Docker build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full local builds took 21-40 min on Apple Silicon (full amd64 emulation). Per-step timing (added here) showed where it went: build:native (C++/JUCE, compiled from scratch every run since build/ was always wiped), the core+plugin clone (~49 shallow clones, ~12.5 min once native was cached), and electron-builder packaging both AppImage and deb against the ~2GB resources/ payload. - Keep build/ between runs instead of wiping it every build (the wrapper always mounts at the stable /workspace, so the path-mismatch concern the wipe guarded against doesn't apply to repeated local runs). CLEAN_BUILD=1 restores the old clean-slate behavior. - Route the C++ compile through ccache (CMakeLists.txt, no-op where ccache isn't installed) backed by a persistent Docker volume, plus volumes for Electron/cmake-js downloads. - Cache the core+plugin clone in a persistent volume: clone_or_update() does an in-place fetch+reset when the cached checkout matches the requested repo, falling back to a fresh clone otherwise (cold cache, repo/ref switch, or a failed fetch) — a cold CI runner behaves exactly like the old plain `git clone`. - FAST_BUILD=1 skips build:native/bundle-python/bundle-binaries when their prior artifacts exist, and packages AppImage-only with no compression (LINUX_TARGETS/BUILDER_COMPRESSION override either independently) — the packaged app is unchanged, only how much of it gets rebuilt/compressed for a throwaway local test. - Print a per-step timing table at the end of every build. None of this changes CI (fresh runner, FAST_BUILD unset) or app behavior. --- .devcontainer/Dockerfile | 15 ++++ CMakeLists.txt | 10 +++ scripts/build-common.sh | 135 ++++++++++++++++++++++++++++------ scripts/build-linux-docker.sh | 28 +++++-- 4 files changed, 160 insertions(+), 28 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 73da719..b64a51b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -31,6 +31,15 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends $PACKAGES && \ rm -rf /var/lib/apt/lists/* +# ── ccache ────────────────────────────────────────────────────────────────── +# Not in .packages/apt.txt (that list is kept in sync with the CI workflow by +# hand). ccache only benefits the local Docker builder, where its cache dir is +# a persistent volume; CMakeLists.txt picks it up via find_program and is a +# no-op where it's absent, so CI needs no change. +RUN apt-get update && \ + apt-get install -y --no-install-recommends ccache && \ + rm -rf /var/lib/apt/lists/* + # ── Node.js ───────────────────────────────────────────────────────────────── RUN . /etc/build.env && \ curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ @@ -52,6 +61,12 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 USER vscode WORKDIR /workspace +# Pre-create the cache dirs that build-linux-docker.sh mounts as named volumes. +# A fresh named volume inherits the mountpoint's ownership from the image, so +# creating these as the vscode user means the volumes are writable without a +# root-owned-mount permission fight (ccache, Electron download, cmake-js headers). +RUN mkdir -p /home/vscode/.ccache /home/vscode/.cache /home/vscode/.cmake-js /home/vscode/.slopsmith-src + # Runtime verification. Workspace isn't mounted at image-build time — # package.json only exists when the container is RUN against a mounted # project, so the Electron version check runs at container start rather diff --git a/CMakeLists.txt b/CMakeLists.txt index 47ddb7f..44772ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,16 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +# Route compiles through ccache when it's installed (the Docker builder ships it +# and mounts a persistent cache volume). No-op on hosts without ccache, so CI +# and native macOS/Windows builds are unaffected. +find_program(CCACHE_PROGRAM ccache) +if(CCACHE_PROGRAM) + message(STATUS "ccache found — using it as compiler launcher: ${CCACHE_PROGRAM}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") +endif() + # cmake-js integration if(DEFINED CMAKE_JS_INC) message(STATUS "Building as Node.js native addon (cmake-js)") diff --git a/scripts/build-common.sh b/scripts/build-common.sh index be1dac3..b4448c6 100644 --- a/scripts/build-common.sh +++ b/scripts/build-common.sh @@ -79,6 +79,29 @@ get_cfg() { python3 "$PARSE_CONFIG" "$CONFIG" "$1"; } # --- Common Build Steps --- +# Clone $authed_url into $dir at $ref, OR update an existing checkout in place so +# a persistent clone dir (Docker volume) is reused across builds instead of +# re-downloading ~49 repos every run. $authed_url carries the (rotating) auth +# token; the stored remote is rewritten to the token-free $plain_url so the reuse +# check stays stable as the token changes each run. Uses `reset --hard` (not +# `clean`) on purpose: the core checkout has the plugin dirs nested under it as +# untracked children, and clean would wipe them. Falls back to a fresh clone when +# the dir is absent, is a different repo, or the fetch fails — so a cold CI runner +# (no cache volume) behaves exactly as a plain `git clone` did. +clone_or_update() { + local authed_url="$1" dir="$2" ref="${3:-}" plain_url="$4" + if [[ -d "$dir/.git" && "$(git -C "$dir" config --get remote.origin.url 2>/dev/null)" == "$plain_url" ]]; then + if git -C "$dir" fetch --depth 1 "$authed_url" ${ref:+"$ref"} 2>/dev/null \ + && git -C "$dir" reset --hard FETCH_HEAD >/dev/null 2>&1; then + return 0 + fi + # fetch/reset failed (deleted branch, corrupt cache) — fall through. + fi + rm -rf "$dir" + git clone --depth 1 ${ref:+--branch "$ref"} "$authed_url" "$dir" || return 1 + git -C "$dir" remote set-url origin "$plain_url" +} + # Clone Slopsmith and plugins (shared across all platforms) clone_slopsmith() { # RUNNER_TEMP on Windows runners is a native Windows path @@ -87,7 +110,10 @@ clone_slopsmith() { # default; if a non-POSIX environment really wants RUNNER_TEMP, it # can pass an explicit clone_dir argument (resolved via cygpath -u # on Windows if needed). - local clone_dir="${1:-/tmp/slopsmith}" + # clone_dir defaults to /tmp/slopsmith (fresh every CI run). Set + # SLOPSMITH_CLONE_DIR to a persistent path (a Docker volume, wired by + # build-linux-docker.sh) to reuse the clone across local builds. + local clone_dir="${1:-${SLOPSMITH_CLONE_DIR:-/tmp/slopsmith}}" # Skip if already set for local development if [[ -n "${SLOPSMITH_DIR:-}" ]] && [[ -d "$SLOPSMITH_DIR" ]]; then @@ -95,13 +121,6 @@ clone_slopsmith() { return 0 fi - # Make the clone re-runnable: a leftover dir from a previous failed - # build would otherwise abort `git clone`. CI runners start fresh so - # this is purely a quality-of-life fix for local re-runs. - if [[ -d "$clone_dir" ]]; then - rm -rf "$clone_dir" - fi - # SLOPSMITH_REF selects the core branch/tag to bundle (set by the # Build workflow's slopsmith_ref input). Defaults to main so local # builds and the push/tag CI paths behave exactly as before. @@ -114,8 +133,10 @@ clone_slopsmith() { local slopsmith_repo="${SLOPSMITH_REPO:-got-feedback/feedback}" local _auth="" [[ -n "${GH_CLONE_TOKEN:-}" ]] && _auth="x-access-token:${GH_CLONE_TOKEN}@" - echo "Cloning Slopsmith repository (${slopsmith_repo} ref: ${slopsmith_ref})..." - git clone --depth 1 --branch "$slopsmith_ref" "https://${_auth}github.com/${slopsmith_repo}.git" "$clone_dir" + # mkdir -p the parent so a volume-mounted clone_dir subpath works. + mkdir -p "$(dirname "$clone_dir")" + echo "Cloning/updating Slopsmith repository (${slopsmith_repo} ref: ${slopsmith_ref})..." + clone_or_update "https://${_auth}github.com/${slopsmith_repo}.git" "$clone_dir" "$slopsmith_ref" "https://github.com/${slopsmith_repo}.git" # Remove broken symlinks from plugins dir find "$clone_dir/plugins" -maxdepth 1 -type l -delete 2>/dev/null || true @@ -199,9 +220,10 @@ clone_slopsmith() { dirname="${dirname#feedback-plugin-}" dirname="${dirname//-/_}" fi - local clone_args=(--depth 1) - [[ -n "$branch" ]] && clone_args+=(--branch "$branch") - if git clone "${clone_args[@]}" "https://${_auth}github.com/${owner_repo}.git" "$dirname" 2>/dev/null; then + # clone_or_update reuses a cached plugin checkout (persistent clone_dir) + # and honors the optional @branch pin via $branch; a fresh dir falls back + # to a full shallow clone. dirname is relative to the plugins/ cwd. + if clone_or_update "https://${_auth}github.com/${owner_repo}.git" "$dirname" "$branch" "https://github.com/${owner_repo}.git" 2>/dev/null; then cloned=$((cloned + 1)) else echo " skipped ${owner_repo}${branch:+@$branch}" @@ -293,6 +315,15 @@ install_npm_deps() { build_native_addons() { echo_step "Building native addons (audio engine)" + # FAST_BUILD reuses the native .node from the previous build (the whole + # C++/JUCE compile — by far the biggest cost under amd64 emulation). Only + # skips when the artifact is actually present, so a FAST_BUILD run with no + # prior full build still compiles instead of shipping a broken app. + if [[ "${FAST_BUILD:-0}" == "1" && -f "$PROJECT_DIR/build/Release/slopsmith_audio.node" ]]; then + echo_warning "FAST_BUILD: reusing existing build/Release/slopsmith_audio.node" + echo "" + return 0 + fi npm run build:native echo_summary "Native addons built" echo "" @@ -307,6 +338,13 @@ bundle_slopsmith() { bundle_python() { mkdir -p "$PROJECT_DIR/resources" + # FAST_BUILD reuses the extracted python-build-standalone runtime (a network + # download + tar extract that never changes between JS-only iterations). + if [[ "${FAST_BUILD:-0}" == "1" && -x "$PROJECT_DIR/resources/python/runtime/bin/python3" ]]; then + echo_summary "FAST_BUILD: reusing existing resources/python/runtime" + echo "" + return 0 + fi bundle_python_impl echo_summary "Python runtime bundled" echo "" @@ -314,6 +352,13 @@ bundle_python() { bundle_binaries() { mkdir -p "$PROJECT_DIR/resources/bin" + # FAST_BUILD reuses bundled binaries (ffmpeg/fluidsynth/vgmstream + their + # dylib chains — includes a network fetch for vgmstream). + if [[ "${FAST_BUILD:-0}" == "1" && -x "$PROJECT_DIR/resources/bin/fluidsynth" ]]; then + echo_summary "FAST_BUILD: reusing existing resources/bin" + echo "" + return 0 + fi bundle_binaries_impl echo_summary "System binaries bundled" echo "" @@ -507,7 +552,29 @@ package_application() { exit 1 ;; esac - npx electron-builder "$builder_platform" --publish never + local builder_args=("$builder_platform") + # FAST_BUILD trims packaging for local iteration, where the shipped + # artifact itself is throwaway: package.json's linux.target builds BOTH + # AppImage and deb (each separately compresses the ~2GB resources/ + # payload), and electron-builder's default asar/7z compression is tuned + # for a real release, not a quick smoke test. Neither changes what's IN + # the app, only how it's packaged for testing — CI/non-fast builds get + # the full untouched config. LINUX_TARGETS/BUILDER_COMPRESSION let a + # caller override either independently of FAST_BUILD. + if [[ "$PLATFORM" == "linux" ]]; then + local targets="${LINUX_TARGETS:-}" + if [[ -z "$targets" && "${FAST_BUILD:-0}" == "1" ]]; then + targets="AppImage" + fi + [[ -n "$targets" ]] && builder_args+=("$targets") + fi + local compression="${BUILDER_COMPRESSION:-}" + if [[ -z "$compression" && "${FAST_BUILD:-0}" == "1" ]]; then + compression="store" + fi + [[ -n "$compression" ]] && builder_args+=("-c.compression=$compression") + + npx electron-builder "${builder_args[@]}" --publish never echo_summary "Application packaged" echo "" } @@ -553,6 +620,27 @@ verify_artifacts() { echo "" } +# Per-step timing. run_timed records "|