diff --git a/content/starter/beethoven-fur_elise.feedpak b/content/starter/beethoven-fur_elise.feedpak new file mode 100755 index 0000000..c06dc67 Binary files /dev/null and b/content/starter/beethoven-fur_elise.feedpak differ diff --git a/server.py b/server.py index 596208e..0952664 100644 --- a/server.py +++ b/server.py @@ -8,6 +8,7 @@ import logging import math import os import secrets +import stat import sys import tempfile import shutil @@ -5576,13 +5577,217 @@ def _get_progression_content() -> dict: return _progression_content +def _copy_builtin_packs( + root: Path, + dest_dir: Path, + sources: list[tuple[str, str]], + label: str, + update_existing: bool = True, +) -> int: + """Symlink-safe, mtime-aware copy of bundled packs into ``dest_dir``. + + ``sources`` is a list of ``(dest_name, rel_source)`` pairs; each source is + resolved under ``root`` (the repo root in dev, ``resources/feedBack`` when + bundled). A pack is copied when its destination is missing. Never deletes + user files; refuses to follow a symlinked seed directory or destination and + refuses to clobber a non-regular destination (any would let a copy escape + ``dest_dir`` or destroy user data). Logs and continues on error. ``label`` + prefixes every log line. + + ``update_existing`` controls what happens when a *regular* destination file + already exists: when True (diagnostic seed) a bundle copy newer than the + destination refreshes it; when False (one-time starter content) an existing + file is always left as-is so the user's copy is never overwritten. + + Returns the number of ``sources`` that are present at their destination + afterwards (freshly seeded, refreshed, or already current) — so callers can + tell whether every pack made it. A skip (missing source, symlink/non-regular + refusal, copy error) does not count. + """ + # Refuse a symlinked seed directory: mkdir(exist_ok=True) would accept it + # and copies would land at the link target, outside the DLC tree. The + # per-file symlink guard below cannot catch this. + if dest_dir.is_symlink(): + log.warning("%s: %s is a symlink, skipping all seeding", label, dest_dir.name) + return 0 + dest_dir.mkdir(parents=True, exist_ok=True) + + # Pin the seed directory by an O_NOFOLLOW fd so a symlink swapped in for + # dest_dir *after* the check above cannot redirect the per-file stat / + # temp-create / replace outside the DLC tree (parent-directory TOCTOU). + # os.replace accepts dir_fd on POSIX even though it isn't listed in + # os.supports_dir_fd, so gate on os.rename (the reliable proxy); platforms + # without dir_fd/O_NOFOLLOW (e.g. Windows) fall back to path-based ops. + dir_fd = None + if ( + hasattr(os, "O_NOFOLLOW") + and hasattr(os, "O_DIRECTORY") + and os.open in os.supports_dir_fd + and os.rename in os.supports_dir_fd + ): + try: + dir_fd = os.open(dest_dir, os.O_RDONLY | os.O_NOFOLLOW | os.O_DIRECTORY) + except OSError as exc: + log.warning("%s: cannot open seed dir %s: %s", label, dest_dir, exc) + return 0 + + try: + present = 0 + for dest_name, rel_source in sources: + source = root / rel_source + if not source.is_file(): + log.warning("%s: source missing, skipping %s (%s)", label, dest_name, source) + continue + + # lstat the destination without following symlinks. Pinned by dir_fd + # this resolves within the real seed dir, immune to a parent swap. + try: + if dir_fd is not None: + dstat = os.lstat(dest_name, dir_fd=dir_fd) + else: + dstat = os.lstat(dest_dir / dest_name) + dest_exists = True + dest_islink = stat.S_ISLNK(dstat.st_mode) + except FileNotFoundError: + dest_exists = False + dest_islink = False + except OSError as exc: + log.warning("%s: cannot stat %s: %s", label, dest_name, exc) + continue + + # Refuse to seed through a symlink at the destination name. + if dest_islink: + log.warning("%s: destination is a symlink, skipping %s", label, dest_name) + continue + + # A non-regular destination (directory, fifo, …) the user placed + # there: never clobber it, and never count it as present — otherwise + # a one-time seed would mark itself done without a real pack on disk. + if dest_exists and not stat.S_ISREG(dstat.st_mode): + log.warning("%s: destination is not a regular file, skipping %s", label, dest_name) + continue + + if dest_exists: + # A regular file is already there. One-time seeds (starter + # content) must never overwrite the user's copy; refreshing + # seeds (diagnostics) replace it only when the bundle is newer. + if not update_existing: + log.info("%s: already present %s", label, dest_name) + present += 1 + continue + try: + src_mtime = source.stat().st_mtime + except OSError as exc: + log.warning("%s: cannot stat source %s: %s", label, source, exc) + continue + if src_mtime <= dstat.st_mtime: + log.info("%s: already present %s", label, dest_name) + present += 1 + continue + action = "updated" + else: + action = "seeded" + + if _write_builtin_pack(source, dest_dir, dest_name, dir_fd): + present += 1 + log.info("%s: %s %s -> %s", label, action, source.name, dest_name) + else: + log.warning("%s: failed to copy %s -> %s/%s", label, source, dest_dir.name, dest_name) + + return present + finally: + if dir_fd is not None: + os.close(dir_fd) + + +def _write_builtin_pack( + source: Path, + dest_dir: Path, + dest_name: str, + dir_fd: int | None, +) -> bool: + """Atomically write ``source`` to ``dest_name`` inside ``dest_dir``. + + Writes to a temp file then ``os.replace()``s onto the final name so a + symlink raced in at the destination is overwritten (rename semantics), not + followed, and a crash never leaves a half-written pack. When ``dir_fd`` is + given, every step is anchored to that fd (O_NOFOLLOW temp create + dir_fd + replace), closing the parent-directory TOCTOU; otherwise falls back to + path-based temp+replace. Returns True on success. Never raises. + """ + # Unique per-attempt name (O_EXCL create) so a crash that orphans a temp + # can't permanently block later seeds via an EEXIST collision. + tmp_name = f".seed-{dest_name}.{os.getpid()}.{secrets.token_hex(4)}.tmp" + try: + src_stat = source.stat() + except OSError as exc: + log.debug("builtin pack: cannot stat source %s: %s", source, exc) + return False + if dir_fd is not None: + tmp_fd = None + try: + tmp_fd = os.open( + tmp_name, + os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, + 0o644, + dir_fd=dir_fd, + ) + with open(source, "rb") as sf, os.fdopen(tmp_fd, "wb") as tf: + tmp_fd = None # fdopen now owns the descriptor + shutil.copyfileobj(sf, tf) + os.replace(tmp_name, dest_name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) + # Preserve the bundle mtime (copyfileobj doesn't) so the mtime-based + # refresh check matches the shutil.copy2 fallback path. Best-effort. + try: + os.utime( + dest_name, + ns=(src_stat.st_atime_ns, src_stat.st_mtime_ns), + dir_fd=dir_fd, + follow_symlinks=False, + ) + except OSError as exc: + log.debug("builtin pack: could not set mtime on %s: %s", dest_name, exc) + return True + except OSError as exc: + log.debug("builtin pack write (dir_fd) failed for %s: %s", dest_name, exc) + if tmp_fd is not None: + try: + os.close(tmp_fd) + except OSError: + pass + try: + os.unlink(tmp_name, dir_fd=dir_fd) + except OSError: + pass + return False + + tmp = None + try: + fd, tmp = tempfile.mkstemp(dir=dest_dir, prefix=".seed-", suffix=".tmp") + os.close(fd) + shutil.copy2(source, tmp) + os.replace(tmp, dest_dir / dest_name) + tmp = None + return True + except OSError as exc: + log.debug("builtin pack write failed for %s: %s", dest_name, exc) + return False + finally: + if tmp is not None: + try: + os.unlink(tmp) + except OSError: + pass + + def _seed_builtin_diagnostic_sloppaks(dlc: Path | None = None) -> None: """Copy bundled diagnostic sloppaks into DLC before library scan. Creates ``DLC_DIR/diagnostics-builtin/`` and copies each bundled sloppak when the destination is missing or older than the repo/bundle source. Never deletes user files or touches manually copied paths (e.g. - ``diagnostics-test/``). Logs and continues on missing source or copy errors. + ``diagnostics-test/``). Re-seeds whenever the destination is missing so the + diagnostic target is always available. Logs and continues on errors. """ try: if dlc is None: @@ -5590,86 +5795,100 @@ def _seed_builtin_diagnostic_sloppaks(dlc: Path | None = None) -> None: if dlc is None: log.debug("Builtin diagnostic seed: no DLC folder configured, skipping") return - - root = _feedBack_server_root() - dest_dir = dlc / _BUILTIN_DIAGNOSTIC_SUBDIR - # Refuse a symlinked seed directory: mkdir(exist_ok=True) would accept - # it and copies would land at the link target, outside the DLC tree. - # The per-file is_symlink() guard below cannot catch this. - if dest_dir.is_symlink(): - log.warning( - "Builtin diagnostic seed: %s is a symlink, skipping all seeding", - _BUILTIN_DIAGNOSTIC_SUBDIR, - ) - return - dest_dir.mkdir(parents=True, exist_ok=True) - - for dest_name, rel_source in _BUILTIN_DIAGNOSTIC_SOURCES: - source = root / rel_source - dest = dest_dir / dest_name - - if not source.is_file(): - log.warning( - "Builtin diagnostic seed: source missing, skipping %s (%s)", - dest_name, - source, - ) - continue - - # Refuse to seed through a symlink. is_file()/stat()/copy2 all - # follow links, so a symlink planted at the destination would let - # the copy redirect outside diagnostics-builtin/ and overwrite an - # arbitrary file the server user can write. Skip and warn; never - # touch the link or its target. - if dest.is_symlink(): - log.warning( - "Builtin diagnostic seed: destination is a symlink, skipping %s/%s", - _BUILTIN_DIAGNOSTIC_SUBDIR, - dest_name, - ) - continue - - if dest.is_file(): - try: - if source.stat().st_mtime <= dest.stat().st_mtime: - log.info( - "Builtin diagnostic seed: already present %s/%s", - _BUILTIN_DIAGNOSTIC_SUBDIR, - dest_name, - ) - continue - action = "updated" - except OSError as exc: - log.warning( - "Builtin diagnostic seed: cannot compare %s and %s: %s", - source, - dest, - exc, - ) - continue - else: - action = "seeded" - - try: - shutil.copy2(source, dest) - log.info( - "Builtin diagnostic seed: %s %s -> %s/%s", - action, - source.name, - _BUILTIN_DIAGNOSTIC_SUBDIR, - dest_name, - ) - except OSError as exc: - log.warning( - "Builtin diagnostic seed: failed to copy %s -> %s: %s", - source, - dest, - exc, - ) + _copy_builtin_packs( + _feedBack_server_root(), + dlc / _BUILTIN_DIAGNOSTIC_SUBDIR, + _BUILTIN_DIAGNOSTIC_SOURCES, + "Builtin diagnostic seed", + ) except Exception: log.warning("Builtin diagnostic seed: unexpected error", exc_info=True) +# Starter content: bundled songs copied into ``DLC_DIR/starter/`` exactly ONCE, +# on first run, as a welcome library so a fresh install isn't empty. Unlike the +# diagnostic seed this is one-time — guarded by a marker in CONFIG_DIR — so if +# the user deletes the starter song it stays gone. ``starter/`` is NOT in the +# library scan carve-out (unlike diagnostics-builtin/ / tutorials-builtin/), so +# seeded packs surface as ordinary library songs. +_BUILTIN_STARTER_SUBDIR = "starter" +_BUILTIN_STARTER_SOURCES: list[tuple[str, str]] = [ + ( + "beethoven-fur_elise.feedpak", + "content/starter/beethoven-fur_elise.feedpak", + ), +] +_STARTER_SEED_MARKER = ".starter-content-seeded" + + +def _seed_builtin_starter_content(dlc: Path | None = None) -> None: + """Copy bundled starter songs into ``DLC_DIR/starter/`` exactly once. + + Guarded by ``CONFIG_DIR/.starter-content-seeded``: the first run with a DLC + folder configured seeds the packs and writes the marker; subsequent runs are + no-ops, so a user who deletes the starter song does not get it back on the + next launch. Symlink-safe; never deletes user files. Logs, never raises. + """ + try: + marker = CONFIG_DIR / _STARTER_SEED_MARKER + # Already seeded? The marker is a sentinel: any existing path there + # (regular file, or a symlink/dir a user deliberately planted to opt + # out) means "done" — lstat so we detect it without following a symlink. + # Worst case of a planted marker is simply no starter content, never a + # data write; the O_EXCL|O_NOFOLLOW create below refuses to write + # *through* a symlink regardless. + try: + os.lstat(marker) + return + except FileNotFoundError: + pass + except OSError as exc: + log.warning("Starter content seed: cannot stat marker %s: %s", marker, exc) + return + if dlc is None: + dlc = _get_dlc_dir() + if dlc is None: + # No DLC yet — leave the marker unwritten so we retry once a + # library folder is configured. + log.debug("Starter content seed: no DLC folder configured, skipping") + return + present = _copy_builtin_packs( + _feedBack_server_root(), + dlc / _BUILTIN_STARTER_SUBDIR, + _BUILTIN_STARTER_SOURCES, + "Starter content seed", + update_existing=False, + ) + # Only mark seeding complete once every starter pack is actually in + # place. If a source was missing or a copy failed, leave the marker + # unwritten so the next launch retries rather than permanently skipping. + if present < len(_BUILTIN_STARTER_SOURCES): + log.info( + "Starter content seed: %d/%d packs present, will retry next launch", + present, + len(_BUILTIN_STARTER_SOURCES), + ) + return + # Record completion with an exclusive, no-follow create so a planted or + # raced symlink at the marker path can't redirect the write outside + # CONFIG_DIR. O_EXCL fails (EEXIST) on any existing path including a + # symlink, so we never write through one. + try: + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(marker, flags, 0o644) + try: + os.write(fd, b"1\n") + finally: + os.close(fd) + except FileExistsError: + pass # already marked (or a non-regular path is squatting) — fine + except OSError as exc: + log.warning("Starter content seed: could not write marker %s: %s", marker, exc) + except Exception: + log.warning("Starter content seed: unexpected error", exc_info=True) + + def _background_scan(): """Scan the library and cache song metadata on startup. Uses a process pool to bypass the GIL for CPU-bound metadata parsing. @@ -5690,6 +5909,7 @@ def _background_scan(): return _seed_builtin_diagnostic_sloppaks(dlc) + _seed_builtin_starter_content(dlc) # Listing can fail on macOS without Full Disk Access, or on Docker if the # path isn't shared. Report the failure explicitly rather than silently diff --git a/tests/test_builtin_starter_seed.py b/tests/test_builtin_starter_seed.py new file mode 100644 index 0000000..66f5b4a --- /dev/null +++ b/tests/test_builtin_starter_seed.py @@ -0,0 +1,183 @@ +"""Tests for one-time builtin starter-content seeding into DLC.""" + +from __future__ import annotations + +import importlib +import sys + +import pytest + + +@pytest.fixture() +def server_mod(tmp_path, monkeypatch, isolate_logging): + monkeypatch.setenv("CONFIG_DIR", str(tmp_path / "config")) + (tmp_path / "config").mkdir() + monkeypatch.delenv("DLC_DIR", raising=False) + sys.modules.pop("server", None) + mod = importlib.import_module("server") + yield mod + + +def _source(server_mod): + return ( + server_mod._feedBack_server_root() + / server_mod._BUILTIN_STARTER_SOURCES[0][1] + ) + + +def _dest(server_mod, dlc): + return ( + dlc + / server_mod._BUILTIN_STARTER_SUBDIR + / server_mod._BUILTIN_STARTER_SOURCES[0][0] + ) + + +def test_seed_creates_starter_content_and_marker(tmp_path, server_mod): + """First run copies the bundled feedpak into starter/ and writes the marker.""" + dlc = tmp_path / "dlc" + dlc.mkdir() + source = _source(server_mod) + if not source.is_file(): + pytest.skip(f"starter source not present in checkout: {source}") + + server_mod._seed_builtin_starter_content(dlc) + + dest = _dest(server_mod, dlc) + assert dest.is_file() + assert dest.stat().st_size == source.stat().st_size + assert (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).is_file() + + +def test_seed_preserves_source_mtime(tmp_path, server_mod): + """The seeded pack keeps the bundle's mtime so the diagnostic refresh check + (source newer than dest -> update) stays correct across both write paths.""" + dlc = tmp_path / "dlc" + dlc.mkdir() + source = _source(server_mod) + if not source.is_file(): + pytest.skip(f"starter source not present in checkout: {source}") + + server_mod._seed_builtin_starter_content(dlc) + + assert _dest(server_mod, dlc).stat().st_mtime_ns == source.stat().st_mtime_ns + + +def test_starter_is_not_carved_out_of_the_library(): + """`starter/` must NOT collide with the diagnostics/tutorials carve-out — + otherwise seeded songs would never appear in the library listing.""" + assert "starter" not in {"diagnostics-builtin", "tutorials-builtin"} + + +def test_seed_runs_only_once_and_respects_deletion(tmp_path, server_mod): + """After the first seed, deleting the song does NOT bring it back: the + marker makes starter seeding a one-time welcome.""" + dlc = tmp_path / "dlc" + dlc.mkdir() + source = _source(server_mod) + if not source.is_file(): + pytest.skip(f"starter source not present in checkout: {source}") + + server_mod._seed_builtin_starter_content(dlc) + dest = _dest(server_mod, dlc) + assert dest.is_file() + + # User removes the starter song. + dest.unlink() + + # A subsequent launch must not re-seed it. + server_mod._seed_builtin_starter_content(dlc) + assert not dest.exists() + + +def test_seed_deferred_until_dlc_configured(tmp_path, server_mod): + """With no DLC folder, seeding is skipped WITHOUT writing the marker, so it + retries once a library folder exists.""" + source = _source(server_mod) + if not source.is_file(): + pytest.skip(f"starter source not present in checkout: {source}") + + # dlc is None and DLC_DIR unset -> _get_dlc_dir() returns None. + server_mod._seed_builtin_starter_content(None) + assert not (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).exists() + + # Now a DLC is configured: the deferred seed runs. + dlc = tmp_path / "dlc" + dlc.mkdir() + server_mod._seed_builtin_starter_content(dlc) + assert _dest(server_mod, dlc).is_file() + assert (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).is_file() + + +def test_seed_refuses_symlinked_seed_directory(tmp_path, server_mod): + """A symlinked starter/ dir is refused so copies can't escape the DLC tree.""" + dlc = tmp_path / "dlc" + dlc.mkdir() + source = _source(server_mod) + if not source.is_file(): + pytest.skip(f"starter source not present in checkout: {source}") + + outside_dir = tmp_path / "outside" + outside_dir.mkdir() + (dlc / server_mod._BUILTIN_STARTER_SUBDIR).symlink_to(outside_dir) + + server_mod._seed_builtin_starter_content(dlc) + + assert list(outside_dir.iterdir()) == [] + # An incomplete seed must NOT write the marker, so a later launch retries. + assert not (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).exists() + + +def test_seed_never_overwrites_an_existing_user_file(tmp_path, server_mod): + """One-time starter seeding must never replace a user's own file at the + destination, even if the bundled pack has a newer mtime.""" + import os as _os + + dlc = tmp_path / "dlc" + dlc.mkdir() + dest = _dest(server_mod, dlc) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(b"user's own edited pack") + _os.utime(dest, (1_000_000, 1_000_000)) # far older than the bundled source + + server_mod._seed_builtin_starter_content(dlc) + + assert dest.read_bytes() == b"user's own edited pack" # untouched + # counted as already-present, so the one-time seed considers itself done + assert (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).is_file() + + +def test_seed_does_not_mark_when_destination_is_a_directory(tmp_path, server_mod): + """A directory sitting at the destination name is neither clobbered nor + counted as present, so the marker stays unwritten and seeding retries.""" + dlc = tmp_path / "dlc" + dlc.mkdir() + source = _source(server_mod) + if not source.is_file(): + pytest.skip(f"starter source not present in checkout: {source}") + + bogus = _dest(server_mod, dlc) + bogus.parent.mkdir(parents=True, exist_ok=True) + bogus.mkdir() # user (or junk) placed a directory where the pack goes + + server_mod._seed_builtin_starter_content(dlc) + + assert bogus.is_dir() # untouched + assert not (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).exists() + + +def test_seed_does_not_mark_when_source_missing(tmp_path, server_mod, monkeypatch): + """If a starter source can't be found, the marker stays unwritten and the + seed is retried on the next launch (rather than permanently skipped).""" + dlc = tmp_path / "dlc" + dlc.mkdir() + monkeypatch.setattr( + server_mod, + "_BUILTIN_STARTER_SOURCES", + [("missing.feedpak", "content/starter/does-not-exist.feedpak")], + ) + + server_mod._seed_builtin_starter_content(dlc) + + assert not (dlc / server_mod._BUILTIN_STARTER_SUBDIR / "missing.feedpak").exists() + assert not (server_mod.CONFIG_DIR / server_mod._STARTER_SEED_MARKER).exists()