mirror of
https://github.com/got-feedBack/feedBack.git
synced 2026-08-11 11:19:24 +00:00
Fix slow library cover loading: serve sloppak art without unpacking + revalidated caching (#534)
* sloppak: read cover without unpacking + serialize/cap zip unpacks Album art for a zip-form sloppak was served by resolve_source_dir(), which unpacks the ENTIRE archive (stems included, ~30 MB) to disk just to read cover.jpg. On the library grid that meant a full extraction per card on scroll. - read_cover_bytes(): opens only the cover member from the zip (or reads the file for dir-form), with zip-slip guarding. ~4 ms vs a full unpack. - resolve_source_dir(): per-file lock + bounded global semaphore so concurrent callers don't rmtree + re-extract the same dest at once (a race), and a burst can't saturate disk/CPU. 8 concurrent calls now dedupe to 1 unpack. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * server: serve sloppak art via read_cover_bytes + cache album-art responses - get_song_art() sloppak branch now reads the cover directly (no full unpack), off-thread via asyncio.to_thread. - All art responses carry Cache-Control: public, max-age=86400. URLs are already cache-busted with ?v=<mtime>, so the browser stops re-fetching every cover on scroll-back; day bound self-heals any URL missing ?v. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * v3 library: lazy-load + async-decode card cover images The grid (24 cards/page) and artist-row thumbnails emitted plain <img> with no loading hint, so a whole page of covers fetched + decoded at once on each scroll batch. Add loading="lazy" decoding="async" to defer off-screen fetches and keep image decode off the main thread. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * address Codex review: zip cover normalization + correct art revalidation Findings from the preflight Codex passes: - sloppak.read_cover_bytes (zip form) read the raw manifest cover string via zf.read(), so a non-canonical name like './cover.jpg' or 'art/../cover.jpg' 404'd. Normalize via safe_join → relative member; reject escape and the degenerate root-collapse case ('.', 'subdir/..') like _unpack_zip does. - Album-art caching is correctness-first: Cache-Control: no-cache plus a strong validator, with real conditional handling (Starlette FileResponse emits an ETag but doesn't evaluate If-None-Match). All three art paths route through _art_conditional/_file_art_response → bodyless 304 on a matching validator. A long immutable max-age was rejected because the frontend ?v=<mtime> buster is only second-resolution and would pin a same-second rewrite. - The sloppak cover is validated by CONTENT (sha1 of the bytes), not a stat: a dir-form sloppak edited in place changes the cover file's mtime but not the directory's, so a dir-stat ETag could emit a stale 304. Content hashing is correct for both dir- and zip-form. get_song_art gained an optional request (internal get_art caller passes none — safe). Adds tests/test_sloppak_cover_art.py pinning read_cover_bytes (canonical, non-canonical, degenerate/escape, dir/zip, webp) and the endpoint's 304 contract incl. the dir-form in-place-edit no-stale-304 regression. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
a7a93e9bef
commit
21997f4b5c
+107
-3
@@ -54,6 +54,27 @@ def is_sloppak(path: Path) -> bool:
|
||||
_source_cache: dict[str, tuple[Path, float, int]] = {}
|
||||
_source_lock = threading.Lock()
|
||||
|
||||
# Full-archive unpacks (zip form) are expensive — they write every stem to
|
||||
# disk. Cap how many run at once so a burst (e.g. many plays queued, or a stray
|
||||
# caller looping the library) can't saturate disk/CPU, and serialize per-file so
|
||||
# two callers never rmtree + re-extract the same dest simultaneously (which
|
||||
# would corrupt the half-written dir the other is reading).
|
||||
_UNPACK_MAX_CONCURRENCY = 2
|
||||
_unpack_semaphore = threading.BoundedSemaphore(_UNPACK_MAX_CONCURRENCY)
|
||||
_unpack_locks: dict[str, threading.Lock] = {}
|
||||
_unpack_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
def _unpack_lock_for(filename: str) -> threading.Lock:
|
||||
"""Return a stable per-file lock so concurrent unpacks of the same sloppak
|
||||
serialize instead of racing on the same destination dir."""
|
||||
with _unpack_locks_guard:
|
||||
lk = _unpack_locks.get(filename)
|
||||
if lk is None:
|
||||
lk = threading.Lock()
|
||||
_unpack_locks[filename] = lk
|
||||
return lk
|
||||
|
||||
|
||||
def _unpack_zip(zip_path: Path, dest: Path) -> None:
|
||||
"""Extract a sloppak zip archive into dest, replacing any previous contents.
|
||||
@@ -126,10 +147,26 @@ def resolve_source_dir(
|
||||
if path.is_dir():
|
||||
resolved = path
|
||||
else:
|
||||
# Zip form — unpack to the cache.
|
||||
# Zip form — unpack to the cache. Serialize per-file (so concurrent
|
||||
# callers don't rmtree + re-extract the same dest at once) and cap
|
||||
# global unpack concurrency (so a burst can't saturate disk/CPU).
|
||||
dest = unpack_cache_root / _safe_id(filename)
|
||||
_unpack_zip(path, dest)
|
||||
resolved = dest
|
||||
with _unpack_lock_for(filename):
|
||||
# Re-check the cache inside the per-file lock — a prior holder may
|
||||
# have just finished unpacking this exact (mtime, size).
|
||||
with _source_lock:
|
||||
cached = _source_cache.get(filename)
|
||||
if (
|
||||
cached
|
||||
and cached[1] == mtime
|
||||
and cached[2] == size
|
||||
and cached[0].exists()
|
||||
):
|
||||
resolved = cached[0]
|
||||
else:
|
||||
with _unpack_semaphore:
|
||||
_unpack_zip(path, dest)
|
||||
resolved = dest
|
||||
|
||||
with _source_lock:
|
||||
_source_cache[filename] = (resolved, mtime, size)
|
||||
@@ -179,6 +216,73 @@ def load_manifest(path: Path) -> dict:
|
||||
return _read_manifest_from_zip(path)
|
||||
|
||||
|
||||
_COVER_MEDIA_TYPES = {
|
||||
".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".png": "image/png", ".webp": "image/webp",
|
||||
}
|
||||
|
||||
|
||||
def _cover_media_type(name: str) -> str:
|
||||
return _COVER_MEDIA_TYPES.get(Path(name).suffix.lower(), "image/jpeg")
|
||||
|
||||
|
||||
def read_cover_bytes(
|
||||
path: Path, manifest: dict | None = None
|
||||
) -> tuple[bytes, str] | None:
|
||||
"""Return ``(image_bytes, media_type)`` for a sloppak's cover, or ``None``.
|
||||
|
||||
Reads ONLY the cover image. For a zipped sloppak this opens the single
|
||||
cover member rather than unpacking the whole archive (stems included), so
|
||||
serving album art on the library grid never triggers a full extraction —
|
||||
the dominant cost behind slow cover loading on scroll.
|
||||
"""
|
||||
try:
|
||||
if manifest is None:
|
||||
manifest = load_manifest(path)
|
||||
except Exception:
|
||||
manifest = {}
|
||||
cover_rel = str((manifest or {}).get("cover") or "cover.jpg")
|
||||
|
||||
if path.is_dir():
|
||||
# Directory form — read the file, guarding against escape.
|
||||
cover_path = (path / cover_rel).resolve()
|
||||
try:
|
||||
cover_path.relative_to(path.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
if cover_path.is_file():
|
||||
try:
|
||||
return cover_path.read_bytes(), _cover_media_type(cover_path.name)
|
||||
except OSError as e:
|
||||
log.warning("sloppak: failed to read cover %r: %s", cover_path, e)
|
||||
return None
|
||||
|
||||
# Zip form — read just the cover member, no unpack. Normalize the manifest
|
||||
# name the way the filesystem would (collapse './' and 'a/../b', backslash →
|
||||
# slash) so a non-canonical-but-valid cover like './cover.jpg' still resolves
|
||||
# to the archive member 'cover.jpg' — matching the old unpack-then-resolve
|
||||
# behavior — and reject zip-slip escape before opening.
|
||||
_zip_root = Path("/_root").resolve()
|
||||
safe = safe_join(_zip_root, cover_rel)
|
||||
# `safe is None` → escape; `safe == _zip_root` → a degenerate name like "."
|
||||
# or "subdir/.." that collapses to the root (member would be "."). Reject
|
||||
# both, mirroring _unpack_zip's degenerate-root guard.
|
||||
if safe is None or safe == _zip_root:
|
||||
log.warning("sloppak: rejected unsafe cover name %r in %r", cover_rel, path)
|
||||
return None
|
||||
member = safe.relative_to(_zip_root).as_posix()
|
||||
try:
|
||||
with zipfile.ZipFile(str(path), "r") as zf:
|
||||
try:
|
||||
data = zf.read(member)
|
||||
except KeyError:
|
||||
return None
|
||||
return data, _cover_media_type(member)
|
||||
except (OSError, zipfile.BadZipFile, RuntimeError) as e:
|
||||
log.warning("sloppak: failed to read cover from zip %r: %s", path, e)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadedSloppak:
|
||||
"""Result of loading a sloppak: the Song object plus stem descriptors."""
|
||||
|
||||
Reference in New Issue
Block a user