From 59bcf338a331cc0cbeab3a66a5e571151033c3e6 Mon Sep 17 00:00:00 2001 From: Matthew Harris Glover Date: Wed, 22 Jul 2026 18:09:46 -0400 Subject: [PATCH] feat(career): host higher venues as opt-in content packs (#1023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(career): host higher venues as opt-in content packs Move the club and arena venue packs (~678 MB of crowd MP4s) out of the bundle and download them on demand, keeping the bar starter bundled so career still works offline. Leans on career's existing pack pipeline (_download_pack: stream -> sha256 -> extract -> validate -> swap), which already degrades gracefully when a pack is absent. - venues.json: club/arena gain `pack` URLs pointing at per-pack, versioned, immutable releases (venue--v, matching the existing venue-arena-v1). Arena's sha256/bytes are the real published asset (verified end-to-end); club is a placeholder until its release is published. - tools/content_packs.py: reusable, reproducible pack build/publish/manifest tool. Byte-identical output for identical media (fixed order/mtime/perms, STORED) so a pack's hash can be known before upload. --local (file://) for offline tests, --publish for the per-pack release. Has a --selfcheck. - .github/workflows/content-packs.yml: workflow_dispatch automation that builds/publishes packs and opens the venues.json manifest-bump PR, so publishing is never a manual checklist. - test: round-trips a tool-built pack through career's real _download_pack. Part of the nightly-slimming effort (feedBack-desktop#122). The desktop bundle change (stop shipping club/arena) is a companion PR. Co-Authored-By: Claude Opus 4.8 * fix(career): don't offer a venue pack until its release is published A committed venues.json entry carries a 0-byte placeholder (and all-zero sha) until its release exists. Previously has_pack was true as soon as a `pack` object was present, so the UI showed a "Download" button that could only fail (the placeholder URL 404s). Gate on a real, publish-stamped size via _pack_published(): the card shows "coming soon" and the download endpoint 404s until the pack is actually published. Caught by a real bundle+runtime smoke. Co-Authored-By: Claude Opus 4.8 * fix(content-packs): address CodeRabbit review on #1023 - workflow: stop interpolating dispatch inputs into Bash (template injection flagged by zizmor). Pass venues/version via env, validate formats, use an argument array. - content_packs: reject top-level files the career downloader would refuse (PACK_FILENAME_RE) before publishing — a stray .DS_Store would otherwise ship and fail _validate_pack_dir for every client. + test. - content_packs: pin ZipInfo.create_system=3 so packs hash identically across Windows/Unix runners (was the documented reproducibility caveat). Co-Authored-By: Claude Opus 4.8 * docs(changelog): note opt-in career venue packs (#122) Signed-off-by: Matthew Harris Glover * docs(content_packs): correct --publish usage in module docstring --publish is a flag (no tag arg) and publish() deliberately omits --clobber; the docstring said otherwise. Signed-off-by: byrongamatos --------- Signed-off-by: Matthew Harris Glover Signed-off-by: byrongamatos Co-authored-by: Claude Opus 4.8 Co-authored-by: byrongamatos --- .github/workflows/content-packs.yml | 90 +++++++++++++ CHANGELOG.md | 6 + plugins/career/routes.py | 11 +- plugins/career/venues.json | 12 +- tests/plugins/career/test_routes.py | 63 ++++++++- tools/content_packs.py | 191 ++++++++++++++++++++++++++++ 6 files changed, 367 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/content-packs.yml create mode 100644 tools/content_packs.py diff --git a/.github/workflows/content-packs.yml b/.github/workflows/content-packs.yml new file mode 100644 index 0000000..ab6d22c --- /dev/null +++ b/.github/workflows/content-packs.yml @@ -0,0 +1,90 @@ +name: Content packs + +# Build & publish opt-in career venue packs as per-pack, versioned, immutable +# releases (convention: tag `venue--v`, asset `-pack-v.zip`), +# then open a PR bumping venues.json url/sha256/bytes. This is automation so +# publishing packs is never a person's manual job (SLIM-NIGHTLY item 1b). +# +# Immutable tags → publishing is a deliberate, versioned act, so this runs on +# manual dispatch (not push): a media change means a new version, a human call. +on: + workflow_dispatch: + inputs: + venues: + description: "Space-separated venue ids to (re)publish, e.g. 'club arena'" + required: true + default: "club" + version: + description: "Pack version N (tag venue--vN). Bump for new media." + required: true + default: "1" + +concurrency: + group: content-packs + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Media lives in-tree today; add `lfs: true` once SLIM-NIGHTLY item 4 + # moves venue-packs/** to Git LFS. + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build & publish packs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Never interpolate dispatch inputs straight into the shell — a crafted + # value would execute on the runner with this job's write token. Pass + # via env, validate the formats, and use a Bash argument array. + VENUES: ${{ github.event.inputs.venues }} + VERSION: ${{ github.event.inputs.version }} + run: | + set -euo pipefail + [[ "$VERSION" =~ ^[1-9][0-9]*$ ]] || { echo "::error::version must be a positive integer"; exit 1; } + [[ "$VENUES" =~ ^[a-z0-9][a-z0-9-]*(\ [a-z0-9][a-z0-9-]*)*$ ]] || { echo "::error::venues must be space-separated venue ids"; exit 1; } + read -r -a venues <<< "$VENUES" + dirs=() + for v in "${venues[@]}"; do + dirs+=("plugins/career/venue-packs/$v") + done + python tools/content_packs.py "${dirs[@]}" \ + --version "$VERSION" \ + --publish \ + --manifest /tmp/packs-manifest.json + cat /tmp/packs-manifest.json + + - name: Apply url/sha256/bytes to venues.json + run: | + python - <<'PY' + import json, pathlib + manifest = json.load(open("/tmp/packs-manifest.json")) + vpath = pathlib.Path("plugins/career/venues.json") + data = json.loads(vpath.read_text()) + for v in data["venues"]: + m = manifest.get(v["id"]) + if m and v.get("pack"): + v["pack"].update(url=m["url"], sha256=m["sha256"], bytes=m["bytes"]) + vpath.write_text(json.dumps(data, indent=4) + "\n") + PY + + - name: Open manifest-bump PR + uses: peter-evans/create-pull-request@v6 + with: + commit-message: "career: refresh venue pack manifest (v${{ github.event.inputs.version }})" + title: "career: refresh venue pack manifest (v${{ github.event.inputs.version }})" + body: | + Automated by the content-packs workflow after publishing + `${{ github.event.inputs.venues }}` v${{ github.event.inputs.version }} + to their `venue--v${{ github.event.inputs.version }}` releases. + Bumps `venues.json` pack url/sha256/bytes to match the uploaded zips. + branch: content-packs/manifest-bump + delete-branch: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 52105d6..6294fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Opt-in career venue packs (#122)** — higher-tier venue crowd media + (`club`, `arena`) is no longer bundled; the app downloads each pack on demand + from its release when you reach the venue (sha256-verified), keeping the + starter `bar` venue bundled for offline play. Trims ~678 MB from the desktop + download; an unpublished pack shows "coming soon" and plays on the standard + stage until its release lands. - **Session-sync relay WebSocket — `/ws/sync/{session_id}` (#1030).** A deliberately dumb JSON fan-out room: a text frame received from one client is forwarded verbatim to every other client on the same session id; the server diff --git a/plugins/career/routes.py b/plugins/career/routes.py index 7ec4a17..f53a510 100644 --- a/plugins/career/routes.py +++ b/plugins/career/routes.py @@ -104,6 +104,13 @@ def _bundled(venue_id): return (_bundled_venue_dir(venue_id) / "manifest.json").is_file() +def _pack_published(pack): + """A remote pack is downloadable only once a publish has stamped its real + size — the committed manifest carries a 0-byte placeholder (and an all-zero + sha) until then, so don't offer a download that can't succeed yet.""" + return bool(pack and (pack.get("bytes") or 0) > 0) + + def _stars(): """(total, per-song dict, detail rows). Accuracy is a 0..1 fraction.""" db = _state["meta_db"] @@ -664,7 +671,7 @@ def setup(app, context): "unlocked": stars_total >= v["star_threshold"], "installed": _installed(v["id"]), "bundled": _bundled(v["id"]), - "has_pack": _bundled(v["id"]) or bool(v.get("pack")), + "has_pack": _bundled(v["id"]) or _pack_published(v.get("pack")), "download": dl, }) return { @@ -934,7 +941,7 @@ def setup(app, context): if venue is None: raise HTTPException(404, "Unknown venue.") pack = venue.get("pack") - if not pack: + if not _pack_published(pack): raise HTTPException(404, "No pack published for this venue yet.") stars_total, _, _ = _stars() if stars_total < venue["star_threshold"]: diff --git a/plugins/career/venues.json b/plugins/career/venues.json index c749a65..623b701 100644 --- a/plugins/career/venues.json +++ b/plugins/career/venues.json @@ -17,14 +17,22 @@ "name": "Velvet Room", "description": "A proper club stage. People actually came to hear you.", "star_threshold": 50, - "pack": null + "pack": { + "url": "https://github.com/got-feedBack/feedBack/releases/download/venue-club-v1/club-pack-v1.zip", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "bytes": 0 + } }, { "id": "arena", "name": "Feedback Arena", "description": "Ten thousand seats. Try not to think about it.", "star_threshold": 150, - "pack": null + "pack": { + "url": "https://github.com/got-feedBack/feedBack/releases/download/venue-arena-v1/arena-pack-v1.zip", + "sha256": "8898c7912c84123559ecfd8b0afd3be19da9a0a4b04aa9dbb5aa1e7f4e3e1669", + "bytes": 351284599 + } } ] } diff --git a/tests/plugins/career/test_routes.py b/tests/plugins/career/test_routes.py index 1830f7a..dcb844c 100644 --- a/tests/plugins/career/test_routes.py +++ b/tests/plugins/career/test_routes.py @@ -83,10 +83,25 @@ def test_download_without_published_pack_404s(client): def test_download_locked_venue_403s(client, monkeypatch): club = career_routes._venue("club") - monkeypatch.setitem(club, "pack", {"url": "http://x/pack.zip", "sha256": "0" * 64}) + monkeypatch.setitem(club, "pack", {"url": "http://x/pack.zip", "sha256": "0" * 64, "bytes": 123}) assert client.post("/api/plugins/career/packs/club/download").status_code == 403 +def test_placeholder_pack_is_not_offered_until_published(client, monkeypatch): + # A committed manifest carries a 0-byte placeholder until its release is + # published. Such a pack must not be offered (has_pack False) and its + # download must 404 — else the UI shows a button that can only fail. + monkeypatch.setattr(career_routes, "_bundled", lambda vid: False) + club = career_routes._venue("club") + monkeypatch.setitem(club, "pack", + {"url": "http://x/c.zip", "sha256": "0" * 64, "bytes": 0}) + by_id = {v["id"]: v for v in client.get("/api/plugins/career/state").json()["venues"]} + assert by_id["club"]["has_pack"] is False # placeholder → not offered + assert by_id["arena"]["has_pack"] is True # arena ships real bytes + # Even forced, an unpublished pack won't start a download. + assert client.post("/api/plugins/career/packs/club/download").status_code == 404 + + def test_bundled_bar_pack_is_installed_and_served(client): state = client.get("/api/plugins/career/state").json() bar = {v["id"]: v for v in state["venues"]}["bar"] @@ -167,9 +182,53 @@ def test_download_worker_end_to_end(client, tmp_path): assert "sha256" in bad["error"] +def test_content_packs_build_roundtrips_through_download(client, tmp_path): + # tools/content_packs.py must produce a zip the real career worker accepts: + # build_pack → manifest_entry → _download_pack → installed. + from tools import content_packs + + src = tmp_path / "bar" + src.mkdir() + for s in career_routes.REQUIRED_LOOPS: + (src / f"{s}.mp4").write_bytes(b"fake-" + s.encode()) + (src / "cheer.mp4").write_bytes(b"fake-cheer") + (src / "manifest.json").write_text(json.dumps({ + "venue": "bar", "version": 1, + "loops": {s: f"{s}.mp4" for s in career_routes.REQUIRED_LOOPS}, + "stingers": {"cheer": "cheer.mp4"}, + })) + out_dir = tmp_path / "packs" + zip_path = out_dir / content_packs.pack_asset("bar", 1) + info = content_packs.build_pack(src, zip_path) + entry = content_packs.manifest_entry(zip_path, zip_path.resolve().as_uri()) + assert entry["sha256"] == info["sha256"] and entry["bytes"] == info["bytes"] + + progress = {"status": "running", "bytes_done": 0, "bytes_total": 0, "error": None} + career_routes._download_pack("bar", entry, progress) + assert progress["status"] == "done", progress["error"] + assert career_routes._installed("bar") + + +def test_content_packs_rejects_files_the_downloader_would_refuse(tmp_path): + # A stray file (e.g. macOS .DS_Store) must fail the build, not get published + # and then break every client's download at _validate_pack_dir. + from tools import content_packs + + src = tmp_path / "bar" + src.mkdir() + (src / "bored.mp4").write_bytes(b"fake") + (src / ".DS_Store").write_bytes(b"junk") + try: + content_packs.build_pack(src, tmp_path / "bar-pack-v1.zip") + except ValueError as e: + assert "downloader will reject" in str(e) + else: + raise AssertionError("build_pack accepted a .DS_Store the downloader rejects") + + def test_double_download_409s(client, monkeypatch): bar = career_routes._venue("bar") - monkeypatch.setitem(bar, "pack", {"url": "http://x/pack.zip", "sha256": "0" * 64}) + monkeypatch.setitem(bar, "pack", {"url": "http://x/pack.zip", "sha256": "0" * 64, "bytes": 123}) # Pretend one is already running. career_routes._state["downloads"]["bar"] = {"status": "running"} assert client.post("/api/plugins/career/packs/bar/download").status_code == 409 diff --git a/tools/content_packs.py b/tools/content_packs.py new file mode 100644 index 0000000..a4e0d9e --- /dev/null +++ b/tools/content_packs.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Build & publish opt-in content packs (career venue media, rig VST slices). + +Flat-zips a pack directory, sha256s it, and emits the ``{url, sha256, bytes}`` +block that the career and rig_builder download paths consume +(``plugins/career/routes.py`` ``_download_pack``). Two modes: + + --local write zips + a file:// manifest (dev/CI/tests; no network) + --publish create/upload each pack's per-pack release; emit release URLs + +The zip is flat (files at the archive root) to satisfy career's zip-slip guard +(``PACK_FILENAME_RE``) and ``_validate_pack_dir``. This module is the reusable +core the content-packs CI workflow calls, so building packs is automation — +never a person's manual job. + +Run ``python tools/content_packs.py --selfcheck`` for the built-in round-trip. +""" + +import argparse +import hashlib +import json +import re +import subprocess +import sys +import zipfile +from pathlib import Path + +REPO = "got-feedBack/feedBack" # where the content-packs release lives (public) + +# Must mirror career's download-time whitelist (plugins/career/routes.py +# PACK_FILENAME_RE). If the builder packs a name the downloader rejects (e.g. a +# stray .DS_Store), the published pack fails _validate_pack_dir for every client. +PACK_FILENAME_RE = re.compile(r"^[a-z0-9_-]{1,64}\.(mp4|webm|mp3|json)$") + + +def build_pack(src_dir: Path, out_zip: Path) -> dict: + """Flat-zip every file directly under src_dir; return {sha256, bytes}. + + Only regular files at the top level are included (venue packs are flat). + Subdirectories are skipped — a nested tree would trip career's zip-slip + guard on download anyway. + + The build is REPRODUCIBLE: identical file contents always yield a + byte-identical zip (fixed name order, fixed mtime, fixed permissions, + ZIP_STORED). So a sha256 computed on any machine matches the zip the CI + workflow or another contributor produces — anyone can precompute the + manifest values without having to be the one who uploads the asset. + """ + files = sorted((p for p in src_dir.iterdir() if p.is_file()), + key=lambda p: p.name) + if not files: + raise ValueError(f"no files to pack in {src_dir}") + bad = [p.name for p in files if not PACK_FILENAME_RE.fullmatch(p.name)] + if bad: + raise ValueError( + f"{src_dir}: files the downloader will reject: {bad} " + f"(allowed: {PACK_FILENAME_RE.pattern})") + out_zip.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(out_zip, "w", zipfile.ZIP_STORED) as zf: + # ZIP_STORED: the media (mp4/mp3) and .vst3 binaries are already + # compressed; deflating just burns CPU for ~0 gain. + for p in files: + # Fixed mtime (the zip epoch, 1980-01-01) + fixed perms so the + # bytes don't depend on the checkout's file timestamps. + info = zipfile.ZipInfo(p.name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_STORED + # Pin create_system: ZipInfo defaults it from the host OS (0 on + # Windows, 3 on Unix), which would otherwise make the same pack + # hash differently across runners. 3 = Unix. + info.create_system = 3 + info.external_attr = 0o644 << 16 + zf.writestr(info, p.read_bytes()) + data = out_zip.read_bytes() + return {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} + + +def manifest_entry(out_zip: Path, url: str) -> dict: + """Pack info as the download-path expects it: {url, sha256, bytes}.""" + return {"url": url, + "sha256": hashlib.sha256(out_zip.read_bytes()).hexdigest(), + "bytes": out_zip.stat().st_size} + + +# Per-pack, versioned, immutable release convention (matches what the team +# already published, e.g. tag `venue-arena-v1` / asset `arena-pack-v1.zip`). +def pack_tag(pack_id: str, version: int) -> str: + return f"venue-{pack_id}-v{version}" + + +def pack_asset(pack_id: str, version: int) -> str: + return f"{pack_id}-pack-v{version}.zip" + + +def pack_url(pack_id: str, version: int, repo: str = REPO) -> str: + return (f"https://github.com/{repo}/releases/download/" + f"{pack_tag(pack_id, version)}/{pack_asset(pack_id, version)}") + + +def publish(pack_id: str, version: int, zip_path: Path, repo: str = REPO) -> None: + """Create the per-pack release if missing, then upload the versioned zip. + + Tags are immutable: a media change means a new version (v1 → v2), never a + re-upload — so no --clobber. gh errors if the asset already exists, which is + the right guard against overwriting a published, referenced pack. + """ + tag = pack_tag(pack_id, version) + if subprocess.run(["gh", "release", "view", tag, "--repo", repo], + capture_output=True).returncode != 0: + subprocess.run( + ["gh", "release", "create", tag, "--repo", repo, "--latest=false", + "--title", f"{pack_id.capitalize()} venue pack v{version}", + "--notes", "Opt-in career venue pack. Not a code release."], + check=True) + subprocess.run( + ["gh", "release", "upload", tag, str(zip_path), "--repo", repo], check=True) + + +def _pack_id(src_dir: Path) -> str: + return src_dir.name + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("src", nargs="*", type=Path, + help="pack source dirs (e.g. plugins/career/venue-packs/club)") + ap.add_argument("--version", type=int, default=1, + help="pack version (tag venue--v); default 1") + ap.add_argument("--local", type=Path, metavar="DIR", + help="write zips here + a file:// manifest.json; no upload") + ap.add_argument("--publish", action="store_true", + help="create/upload the per-pack release; emit release URLs") + ap.add_argument("--manifest", type=Path, + help="write the {id: {url,sha256,bytes}} map here (default: stdout)") + ap.add_argument("--selfcheck", action="store_true", help="run the round-trip demo and exit") + args = ap.parse_args(argv) + + if args.selfcheck: + return _selfcheck() + if not args.src or (not args.local and not args.publish): + ap.error("need one or more src dirs and either --local or --publish") + + out_dir = args.local if args.local else Path(args.src[0]).parent / "_packs" + manifest = {} + for src in args.src: + pid = _pack_id(src) + zip_path = out_dir / pack_asset(pid, args.version) + build_pack(src, zip_path) + if args.publish: + publish(pid, args.version, zip_path) + url = pack_url(pid, args.version) + else: + url = (out_dir.resolve() / zip_path.name).as_uri() + manifest[pid] = manifest_entry(zip_path, url) + + out = json.dumps(manifest, indent=2) + if args.manifest: + args.manifest.write_text(out + "\n", encoding="utf-8") + else: + print(out) + return 0 + + +def _selfcheck() -> int: + """Build a pack and confirm build_pack/manifest_entry agree on the digest.""" + import tempfile + with tempfile.TemporaryDirectory() as td: + td = Path(td) + src = td / "bar" + src.mkdir() + (src / "manifest.json").write_text('{"venue":"bar"}') + (src / "bored.mp4").write_bytes(b"\x00fake-video") + zip_path = td / pack_asset("bar", 1) + info = build_pack(src, zip_path) + # Reproducible: a second build (into a different path) is byte-identical. + info2 = build_pack(src, td / "again.zip") + assert info2["sha256"] == info["sha256"], "build is not reproducible" + entry = manifest_entry(zip_path, pack_url("bar", 1)) + assert entry["sha256"] == info["sha256"], "digest mismatch" + assert entry["bytes"] == info["bytes"] + assert entry["url"] == ( + f"https://github.com/{REPO}/releases/download/venue-bar-v1/bar-pack-v1.zip") + # Round-trip: the zip must be flat (names == basenames). + with zipfile.ZipFile(zip_path) as zf: + names = zf.namelist() + assert set(names) == {"manifest.json", "bored.mp4"}, names + print("content_packs selfcheck: ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main())