diff --git a/CHANGELOG.md b/CHANGELOG.md index 35092fd..cc3329a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the second slot. The `_demo_mode_guard` middleware still blocks all four moved write routes with 403, and `Query(...)` validation still 422s — both checked against a running server. `server.py`: **9,445 → 9,386 lines**. +- **`lib/enrichment.py` — the metadata-enrichment subsystem leaves `server.py` (R3, move-only).** MusicBrainz / Cover-Art-Archive / AcoustID transport, the match-scorer glue, and the background enrichment worker (~930 lines, 61 defs) move out as one cohesive unit. Bodies are verbatim; the only changes are seam reads — `meta_db`/`config_dir`/`sloppak_cache_dir`/`art_cache_dir` and the two shared art helpers (`song_pack_art_exists`, `art_override_paths`, which stay in `server.py` for the art/delete routes) are reached through `appstate` at call time, and the User-Agent VERSION lookup is corrected for the module's new depth. `server.py` drives the worker through the module (`import enrichment`; the routes + scan lifecycle call `enrichment.X`); tests that faked the network on `server` now patch the same names on `enrichment` (module attribute resolved at call time, so one `setattr` reaches both the routes and the worker's internal callers). Acyclic — `enrichment` imports no `server`. Route table byte-identical; full suite green. `server.py`: 6,917 → 5,988. - **`appstate.py` — the router seam (R3).** Route modules moving out of `server.py` need `meta_db` and friends but must not `import server`, or the import graph goes circular the moment `server` imports them back. So `server.py` keeps *constructing* diff --git a/docs/size-exemptions.md b/docs/size-exemptions.md index daac2fa..9a52c36 100644 --- a/docs/size-exemptions.md +++ b/docs/size-exemptions.md @@ -55,8 +55,8 @@ without a *signed* exemption" is unenforceable. ## Planned, NOT exempt (owned by split plans — listed so nothing falls between states) core `static/app.js` (11,852) · `static/highway.js` (4,168, whole file) · `server.py` -(6,917 — was 14,037; ratcheted by the R3 `MetadataDB` + `AudioEffectsMappingDB` -extractions and fifteen `routers/` modules) · +(5,988 — was 14,037; ratcheted by the R3 `MetadataDB` + `AudioEffectsMappingDB` +extractions and fifteen `routers/` modules; the ~930-line metadata-enrichment subsystem — MB/CAA/AcoustID transport, matcher, background worker — now lives in `lib/enrichment.py`) · `lib/metadata_db.py` (4,373 — new in R3; the `MetadataDB` class alone is 4,018 lines and is a monolith in its own right, to be split per-table once the router train lands) · `static/v3/songs.js` (4,134) · `static/capabilities/audio-session.js` diff --git a/lib/appstate.py b/lib/appstate.py index 526b4b5..7181dd3 100644 --- a/lib/appstate.py +++ b/lib/appstate.py @@ -92,6 +92,12 @@ audio_cache_dir = None get_progression_content = None builtin_diagnostic_filename = None running_version = None +# Art helpers that stay in server.py (shared with the art/delete routes) but are +# also called by the enrichment worker in lib/enrichment.py — injected as +# callables to keep enrichment acyclic. art_cache_dir is server's ART_CACHE_DIR. +art_cache_dir = None +song_pack_art_exists = None +art_override_paths = None _SLOTS = frozenset({ "meta_db", "audio_effect_mappings", "tuning_providers", @@ -99,6 +105,7 @@ _SLOTS = frozenset({ "static_dir", "sloppak_cache_dir", "audio_cache_dir", "get_progression_content", "builtin_diagnostic_filename", "running_version", + "art_cache_dir", "song_pack_art_exists", "art_override_paths", }) diff --git a/lib/enrichment.py b/lib/enrichment.py new file mode 100644 index 0000000..a6431b4 --- /dev/null +++ b/lib/enrichment.py @@ -0,0 +1,1103 @@ +"""The metadata-enrichment subsystem: MusicBrainz/AcoustID/Cover-Art-Archive +transport, the match scorer glue, and the background enrichment worker (R3). + +Moved verbatim out of server.py. The only changes are the seam reads — the +singletons and the two shared art helpers that stay in server.py are reached +through appstate at call time: + + meta_db -> appstate.meta_db + CONFIG_DIR -> appstate.config_dir + SLOPPAK_CACHE_DIR -> appstate.sloppak_cache_dir + ART_CACHE_DIR -> appstate.art_cache_dir + _song_pack_art_exists / _art_override_paths -> appstate. + the User-Agent VERSION lookup: Path(__file__).parent -> .resolve().parents[1] + (lib/enrichment.py -> app root; VERSION ships at the app root everywhere). + +server.py drives the worker through the public names here (import enrichment; +enrichment._kick_enrich(), enrichment._enrich_thread, the routes call the +matchers/transport). Tests that faked the network on `server` now patch the +same names on `enrichment` (the module attribute is resolved at call time, so a +setattr(enrichment, "_mb_http_get", ...) reaches both the routes and the +worker's internal callers). +""" + +import json +import os +import re +import threading +import time +from pathlib import Path + +from fastapi.responses import JSONResponse + +import acoustid_match +import appstate +import loosefolder as loosefolder_mod +import mb_match +import sloppak as sloppak_mod +from appconfig import _load_config +from dlc_paths import _get_dlc_dir, _resolve_dlc_path +from env_compat import env_flag_compat as _env_flag +from metadata_db import _artist_title_from_filename + +import logging +log = logging.getLogger("feedBack.server") + +_enrich_thread: threading.Thread | None = None + + +_enrich_kick_lock = threading.Lock() + + +_enrich_pending_pass = False + + +# processed = phase-1 stubs stamped this pass (legacy field). total/matched = +# the phase-2 MATCHING progress the "Refresh Metadata" batch bar reads (the +# slow, rate-limited part worth a progress readout); current = the song being +# matched right now, which drives the per-tile "working" badge. +_enrich_status = {"running": False, "processed": 0, "last_pass_at": None, + "total": 0, "matched": 0, "current": None} + + +# Cooperative cancel for the Stop button: the matching/art loops check it +# between songs (an in-flight ≤1/s lookup can't be interrupted, but no new one +# is started). Set by /api/enrichment/cancel, cleared when a fresh pass kicks. +_enrich_cancel = threading.Event() + + +# Minimum spacing between EXTERNAL lookups (design: ≤1 req/s + local cache). +_ENRICH_MIN_INTERVAL = 1.1 + + +_enrich_last_fetch = 0.0 + + +# Serializes throttling across the background daemon thread AND the sync +# /api/enrichment/search route (FastAPI runs sync routes in a threadpool). +_enrich_throttle_lock = threading.Lock() + + +def _enrichment_art_dir() -> Path: + """The size-capped art cache dir (populated by the Cover Art slice; the + LRU cap policy lands with it). Under appstate.config_dir so Settings backup/restore + and the docker volume already cover it.""" + d = appstate.config_dir / "art_cache" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _enrich_throttle(): + """Block until an external lookup is allowed. Matchers MUST call this + before every network request — and must NOT hold appstate.meta_db._lock across the + request (fetch outside the lock, write inside).""" + global _enrich_last_fetch + # Hold the lock across the read, sleep, and write so concurrent callers + # serialize instead of all reading the same stale timestamp and firing + # together (which would burst past MusicBrainz's 1 req/s limit). + with _enrich_throttle_lock: + wait = _ENRICH_MIN_INTERVAL - (time.monotonic() - _enrich_last_fetch) + if wait > 0: + time.sleep(wait) + _enrich_last_fetch = time.monotonic() + + +class EnrichTransportError(Exception): + """Network-level enrichment failure — offline, DNS, MusicBrainz down or + rate-limiting. Pauses the current pass (rows keep their state and no + attempt is consumed); the next kick (scan-complete / the 5-min periodic + rescan) retries naturally.""" + + +_MB_API_ROOT = "https://musicbrainz.org/ws/2" + + +_enrich_ua_cache: str | None = None + + +def _enrich_user_agent() -> str: + """MusicBrainz etiquette requires a real identifying User-Agent + (app/version + contact URL); anonymous defaults get throttled/blocked.""" + global _enrich_ua_cache + if _enrich_ua_cache is None: + version = "unknown" + try: + vf = Path(__file__).resolve().parents[1] / "VERSION" + if vf.exists(): + version = vf.read_text().strip() or "unknown" + except (OSError, UnicodeDecodeError): + pass + _enrich_ua_cache = f"feedBack/{version} (https://github.com/got-feedback/feedBack)" + return _enrich_ua_cache + + +def _enrich_network_enabled() -> bool: + """False = the matcher runs local-only (hash stamping, cache copies) and + never opens a socket. FEEDBACK_ENRICH_OFFLINE is the explicit user + kill-switch (privacy / air-gapped installs); FEEDBACK_SKIP_STARTUP_TASKS + marks the test/CI environment, where pytest must never reach the network + no matter what a test triggers.""" + return not (_env_flag("FEEDBACK_ENRICH_OFFLINE") + or _env_flag("FEEDBACK_SKIP_STARTUP_TASKS")) + + +def _mb_http_get(path: str, params: dict) -> dict | None: + """The ONE place enrichment touches the network (tests fake exactly this + seam). Throttled (≤1 req/s via _enrich_throttle), identified (real + User-Agent), offline-guarded. Returns the parsed JSON body, or None for + a 404 lookup; raises EnrichTransportError for anything network-shaped. + NEVER call this while holding appstate.meta_db._lock — fetch outside, write + inside.""" + if not _enrich_network_enabled(): + raise EnrichTransportError("enrichment network disabled") + import requests # declared in requirements.txt; lazy so tests never need it + _enrich_throttle() + try: + resp = requests.get( + f"{_MB_API_ROOT}/{path.lstrip('/')}", + params={**params, "fmt": "json"}, + headers={"User-Agent": _enrich_user_agent()}, + timeout=10, + ) + except requests.RequestException as e: + raise EnrichTransportError(str(e)) from e + if resp.status_code == 404: + return None + if resp.status_code == 503: + # MusicBrainz signals rate-limit pressure with 503 — back the whole + # pass off rather than hammering on. + raise EnrichTransportError("musicbrainz 503 (rate limited)") + if resp.status_code != 200: + raise EnrichTransportError(f"musicbrainz HTTP {resp.status_code}") + try: + return resp.json() + except ValueError as e: + raise EnrichTransportError("bad JSON from musicbrainz") from e + + +def _mb_search_recordings(artist, title, limit: int = 12) -> list[dict]: + """Text search (tier 2–4): denoised Lucene query over /recording. The strict + query drops live-only recordings and the ranker rewards the studio take, so a + slightly larger default result set gives the re-ranker room to surface the + canonical version. + + Runs the strict field-phrase query first (high precision); if it finds + nothing, retries ONCE with a loose term query. The strict phrase only matches + MusicBrainz's *primary* artist/title, so a recording stored under a non-Latin + primary name (大橋純子) whose romanized form ("Junko Ohashi") is only an alias + is invisible to it — the loose query searches aliases and rescues it. The + retry spends a second throttled request only on a miss; results are re-scored + by rank_candidates, so the looser recall doesn't lower match quality + (auto-accept still needs the per-field floors).""" + query = mb_match.build_recording_query(artist, title) + cands: list[dict] = [] + if query: + body = _mb_http_get("recording", {"query": query, "limit": limit}) + cands = mb_match.parse_search_response(body or {}) + if not cands: + loose = mb_match.build_recording_query(artist, title, loose=True) + if loose and loose != query: + body = _mb_http_get("recording", {"query": loose, "limit": limit}) + cands = mb_match.parse_search_response(body or {}) + return cands + + +def _mb_search_release_groups(query: str, limit: int = 8) -> list[dict]: + """Text search /release-group for the Change-cover picker: albums matching a + free query, each mapped to its Cover Art Archive front thumb. One request; + tiles whose CAA art is missing self-hide client-side (front-250 404s). Lets a + cover be found even for a song with no metadata match (the city-pop pile).""" + q = (query or "").strip() + if not q: + return [] + body = _mb_http_get("release-group", {"query": q, "limit": limit}) + out: list[dict] = [] + for rg in ((body or {}).get("release-groups") or []): + rid = rg.get("id") + if not rid: + continue + # artist-credit is a list of {name, joinphrase, artist} (joinphrase glues + # collaborations) — reconstruct the credited name. + artist = "".join( + (c.get("name", "") + c.get("joinphrase", "")) if isinstance(c, dict) else str(c) + for c in (rg.get("artist-credit") or []) + ).strip() + title = rg.get("title") or "" + year = (rg.get("first-release-date") or "")[:4] + out.append({ + "id": rid, + "label": " · ".join(x for x in (title, artist, year) if x) or title or "Cover", + "thumb_url": f"https://coverartarchive.org/release-group/{rid}/front-250", + }) + return out + + +_ACOUSTID_MAX_UPLOAD_BYTES = 256 * 1024 * 1024 # 256 MB — an uncompressed master + + +def _fpcalc_bin() -> str | None: + """Locate the Chromaprint `fpcalc` binary: $FPCALC override, else PATH.""" + import shutil + cand = os.environ.get("FPCALC") + if cand and Path(cand).exists(): + return cand + return shutil.which("fpcalc") + + +def _acoustid_settings() -> "tuple[bool, str]": + """(enabled, api_key) for AcoustID, resolved from settings with an env-var + fallback for the key. Opt-in: `acoustid_enabled` defaults off. The key lives + in settings so a user can set it themselves in the UI; $ACOUSTID_API_KEY is a + server-wide fallback for a headless deploy.""" + cfg = _load_config(appstate.config_dir / "config.json") or {} + enabled = cfg.get("acoustid_enabled", False) is True + key = cfg.get("acoustid_api_key") + if not isinstance(key, str) or not key.strip(): + key = os.environ.get("ACOUSTID_API_KEY", "") + return enabled, (key or "").strip() + + +def _acoustid_available() -> bool: + """True only when the user opted in, a key is set (settings or env), the + network is on, AND fpcalc exists.""" + enabled, key = _acoustid_settings() + return (enabled + and _enrich_network_enabled() + and acoustid_match.is_configured(key) + and _fpcalc_bin() is not None) + + +def _fpcalc(path: str) -> "tuple[int, str] | None": + """Fingerprint a local audio file → (duration_seconds, fingerprint). None on + any failure (missing binary/file, decode error, timeout).""" + binp = _fpcalc_bin() + if not binp or not Path(path).exists(): + return None + import subprocess + import json as _json + try: + pr = subprocess.run([binp, "-json", str(path)], + capture_output=True, timeout=30) + except Exception: + return None + if pr.returncode != 0: + return None + try: + data = _json.loads(pr.stdout.decode("utf-8", "replace")) + dur = int(round(float(data.get("duration")))) + fp = str(data.get("fingerprint") or "") + except Exception: + return None + if not fp or dur <= 0: + return None + return dur, fp + + +def _acoustid_lookup(duration: int, fingerprint: str) -> list[dict]: + """Look a fingerprint up on AcoustID → candidate dicts (mb_match shape). + Throttled + offline-guarded like the MusicBrainz path. [] when unavailable + or no hit; raises EnrichTransportError for network-shaped failures.""" + _, key = _acoustid_settings() + if not key or not _enrich_network_enabled(): + return [] + import requests + _enrich_throttle() + try: + # POST, not GET: a fingerprint is multi-KB (a 3.5-min track is ~3.5k + # chars), so a GET crams it into the URL and a long song overflows the + # server's URL limit → a spurious failure. AcoustID accepts the same + # params form-encoded in the body. + resp = requests.post( + f"{acoustid_match.ACOUSTID_API_ROOT}/lookup", + data={ + "client": key, "format": "json", + "meta": acoustid_match.LOOKUP_META, + "duration": duration, "fingerprint": fingerprint, + }, + headers={"User-Agent": _enrich_user_agent()}, + timeout=10, + ) + except requests.RequestException as e: + raise EnrichTransportError(str(e)) from e + if resp.status_code == 429: + raise EnrichTransportError("acoustid 429 (rate limited)") + if resp.status_code != 200: + raise EnrichTransportError(f"acoustid HTTP {resp.status_code}") + try: + body = resp.json() + except ValueError as e: + raise EnrichTransportError("bad JSON from acoustid") from e + return acoustid_match.parse_lookup_response(body) + + +def _identify_by_fingerprint(path: str) -> list[dict]: + """fpcalc + AcoustID lookup for a local audio file. [] if fingerprinting is + unavailable, the file can't be read, or nothing matched. Available to the + library-enrichment pipeline as well as the /identify endpoint.""" + if not _acoustid_available(): + return [] + fp = _fpcalc(path) + if not fp: + return [] + return _acoustid_lookup(fp[0], fp[1]) + + +def _acoustid_gate() -> "JSONResponse | None": + """Shared availability gate for the identify endpoints: None when ready, + else a 412 needs_setup (opt-in off / no key → the UI re-prompts) or a 503 + (set up but fpcalc/network missing). Never lets a caller pretend a + fingerprint ran.""" + if _acoustid_available(): + return None + enabled, key = _acoustid_settings() + if not enabled or not key: + return JSONResponse( + {"error": "audio fingerprinting not set up", "needs_setup": True, + "detail": "Turn on AcoustID and add a free API key to identify by audio — " + "it reads the recording itself, far more reliable than text search."}, + status_code=412) + return JSONResponse( + {"error": "audio fingerprinting unavailable", "needs_setup": False, + "detail": "the fpcalc (Chromaprint) binary was not found on the server"}, + status_code=503) + + +def _song_audio_file(filename: str) -> "str | None": + """Resolve a LIBRARY song (by filename/id) to a local master-audio file for + fingerprinting: the full-mix `original_audio` extracted from a sloppak, or a + loose folder's audio. None when the song can't be found or ships no full-mix + audio (some packs carry only stems). Mirrors serve_sloppak_file's containment + guards so a crafted filename can't read outside DLC_DIR / the pack.""" + dlc = _get_dlc_dir() + if not dlc: + return None + resolved = _resolve_dlc_path(dlc, filename) + if resolved is None or not resolved.exists(): + return None + if sloppak_mod.is_sloppak(resolved): + try: + canon = resolved.relative_to(dlc.resolve()).as_posix() + except ValueError: + return None + rel = (sloppak_mod.load_manifest(resolved) or {}).get("original_audio") + if not isinstance(rel, str) or not rel.strip(): + return None + src = sloppak_mod.get_cached_source_dir(canon) + if src is None: + try: + src = sloppak_mod.resolve_source_dir(canon, dlc, appstate.sloppak_cache_dir) + except Exception: + return None + target = (src / rel.strip()).resolve() + try: + target.relative_to(src.resolve()) + except ValueError: + return None + return str(target) if target.is_file() else None + try: + audio = loosefolder_mod.find_audio(resolved) + except Exception: + audio = None + return str(audio) if audio and Path(str(audio)).is_file() else None + + +def _mb_lookup_recording(mbid: str) -> dict | None: + """Direct lookup for a manifest-carried recording MBID (tier 0).""" + body = _mb_http_get( + f"recording/{mbid}", + {"inc": "artist-credits+releases+release-groups+isrcs+genres"}) + return mb_match.parse_recording_doc(body) if body else None + + +def _mb_lookup_isrc(isrc: str) -> list[dict]: + """Recordings registered under a manifest-carried ISRC (tier 1).""" + body = _mb_http_get( + f"isrc/{isrc}", {"inc": "artist-credits+releases+release-groups"}) + if not body: + return [] + docs = body.get("recordings") or [] + return [c for c in (mb_match.parse_recording_doc(d) for d in docs) if c] + + +# Strict shapes for the manifest's optional identity keys (feedpak spec §5.1). +# Validated before use — the mbid is interpolated into a URL path, so junk or +# hostile manifest values must never reach the request line. +_MBID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + + +_ISRC_RE = re.compile(r"^[A-Z]{2}[A-Z0-9]{3}[0-9]{7}$") + + +# ── Alias-aware scoring ─────────────────────────────────────────────────────── +# MusicBrainz stores many artists under a non-Latin PRIMARY name (大橋純子) with +# the romanized form ("Junko Ohashi") only as an ALIAS. A recording search +# returns the primary name in its artist-credit, never the aliases — so scoring +# a romanized reference against the primary gives 0 and the match can't confirm. +# We fetch the artist's aliases (one throttled lookup, process-cached) and hand +# them to the scorer, but ONLY for a promising near-miss (title already agrees, +# artist doesn't) so a normal pass spends no extra requests. +_ALIAS_ENRICH_MAX = 3 # cap alias lookups per song/search (each is ≤1/s) + + +_artist_alias_cache: dict[str, list[str]] = {} + + +def _mb_artist_aliases(artist_id: str) -> list[str]: + """Romanized/alternate names for a MusicBrainz artist, process-cached (an + artist recurs across a whole discography, so a library of one artist costs + ONE lookup). Returns [] for an unknown/aliasless artist. Raises + EnrichTransportError on a network failure so the caller pauses the pass + (nothing is cached on failure → retried next pass).""" + aid = str(artist_id or "") + if aid in _artist_alias_cache: + return _artist_alias_cache[aid] + if not _MBID_RE.match(aid): + return [] + body = _mb_http_get(f"artist/{aid}", {"inc": "aliases"}) + names: list[str] = [] + if body: + sort_name = str(body.get("sort-name") or "").strip() + if sort_name: + names.append(sort_name) # often the romanized form for JP artists + for al in body.get("aliases") or []: + if isinstance(al, dict) and al.get("name"): + names.append(str(al["name"])) + seen: set[str] = set() + out: list[str] = [] + for n in names: + k = n.casefold() + if k and k not in seen: + seen.add(k) + out.append(n) + out = out[:12] + _artist_alias_cache[aid] = out + return out + + +def _alias_enrich(ref: dict, cands: list[dict]) -> None: + """Attach `artist_aliases` in place to candidates that look like the + non-Latin-primary case — title agrees with the reference but the primary + artist doesn't — so the scorer can confirm them via a romanized alias. + Bounded by _ALIAS_ENRICH_MAX + the process cache; a no-op when the + reference has no artist or nothing is aliasable.""" + ref_artist = (ref.get("artist") or "").strip() + if not ref_artist: + return + spent = 0 + for c in cands: + if spent >= _ALIAS_ENRICH_MAX: + break + if not isinstance(c, dict) or c.get("artist_aliases") is not None: + continue + aid = c.get("artist_id") + if not aid: + continue + # Only spend a lookup on a promising near-miss: the title already + # matches, but the primary artist doesn't (that's the alias signature). + if mb_match.similarity(ref.get("title"), c.get("title")) < mb_match.AUTO_TITLE_MIN: + continue + if mb_match.similarity(ref_artist, c.get("artist"), artist=True) >= mb_match.AUTO_ARTIST_MIN: + continue + c["artist_aliases"] = _mb_artist_aliases(aid) # cached; attach [] to avoid refetch + spent += 1 + + +def _manifest_exact_ids(filename: str) -> dict: + """Optional `mbid`/`isrc` from the pack manifest — the spec's additive + identity keys. Feature-detected: packs published before that spec + revision simply lack them and fall through to text matching. READ-only: + enrichment never writes anything into pack files.""" + try: + dlc = _get_dlc_dir() + if not dlc: + return {} + p = _resolve_dlc_path(dlc, filename) + if p is None or not p.exists() or not sloppak_mod.is_sloppak(p): + return {} + manifest = sloppak_mod.load_manifest(p) or {} + except Exception: + return {} + out = {} + mbid = str(manifest.get("mbid", "") or "").strip().lower() + if _MBID_RE.match(mbid): + out["mbid"] = mbid + isrc = str(manifest.get("isrc", "") or "").strip().upper() + # Spec 1.14.0: the stored form is the bare 12-char code, but ISRCs + # circulate hyphenated in the wild (US-ABC-24-00001) — the separators + # are presentation, not part of the code, so a hand-authored display + # form still matches (consumers SHOULD strip before comparing). + isrc = isrc.replace("-", "").replace(" ", "") + if _ISRC_RE.match(isrc): + out["isrc"] = isrc + return out + + +# Failed-row retry backoff: 1 h after the first failed attempt, doubling per +# attempt, capped at a week — a permanently-unmatchable obscure chart must +# not re-hammer MusicBrainz on every scan kick. +_ENRICH_BACKOFF_BASE = 3600.0 + + +_ENRICH_BACKOFF_CAP = 7 * 86400.0 + + +def _enrich_backoff_elapsed(attempts, last_attempt_at, now: float) -> bool: + if not last_attempt_at: + return True + delay = min(_ENRICH_BACKOFF_BASE * (2 ** max(0, int(attempts or 1) - 1)), + _ENRICH_BACKOFF_CAP) + return (now - float(last_attempt_at)) >= delay + + +# Review tier keeps a short ranked candidate list for the drawer; more than a +# handful is noise the user has to scroll past. +_ENRICH_MAX_CANDIDATES = 5 + + +# ── Cover art (R3/P9) ───────────────────────────────────────────────────────── +# The art cache dir (appstate.config_dir/art_cache) holds two kinds of file: +# {safe_name}.png / .gif — USER OVERRIDES (upload or URL-fetch; never +# evicted, removed only with the song or by the +# explicit remove-override route) +# caa_{release_mbid}.jpg — COVER ART ARCHIVE fetches, keyed by release so +# every chart of the same release shares one file; +# size-capped LRU (evictions reset the enrichment +# rows so a later pass may re-fetch) +_CAA_CACHE_CAP_BYTES = 200 * 1024 * 1024 + + +# Per-cover cap on a single CAA fetch. The 500px thumbnail is normally tens of +# KB; this bounds any one response independently of the aggregate LRU cap so a +# single oversized (or misbehaving) release can't blow up memory/disk. +_CAA_MAX_BYTES = 10 * 1024 * 1024 + + +# A release MBID is a UUID; before interpolating it into a cache-file path we +# require a conservative token (alphanumerics, hyphen, underscore only) so no +# separator or '.' can ever appear — blocks path traversal. Defence in depth: +# cheap even though the DB only ever holds MusicBrainz UUIDs. (Distinct name +# from the strict recording-MBID _MBID_RE above — this only gates a filename.) +_CAA_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") + + +def _caa_http_get(release_id: str) -> bytes | None: + """Fetch a release's front cover from the Cover Art Archive — the one + network seam of the art layer (tests fake exactly this). Same etiquette + as the MusicBrainz client: throttled, identified, offline-guarded. + Returns the image bytes, None when the release has no cover (404), and + raises EnrichTransportError for anything network-shaped.""" + if not _enrich_network_enabled(): + raise EnrichTransportError("enrichment network disabled") + import requests + _enrich_throttle() + try: + with requests.get( + f"https://coverartarchive.org/release/{release_id}/front-500", + headers={"User-Agent": _enrich_user_agent()}, + timeout=15, allow_redirects=True, stream=True, + ) as resp: + if resp.status_code == 404: + return None + if resp.status_code != 200: + raise EnrichTransportError(f"cover art archive HTTP {resp.status_code}") + # Stream with a per-file cap so a huge response never fully downloads. + data = b"" + for chunk in resp.iter_content(65536): + data += chunk + if len(data) > _CAA_MAX_BYTES: + # Not network-shaped: settle just this row as 'error' (the + # art loop's generic handler) rather than pausing the pass. + raise ValueError("cover art exceeds size cap") + return data + except requests.RequestException as e: + raise EnrichTransportError(str(e)) from e + + +def _caa_release_index(release_id: str) -> dict | None: + """Fetch a release's Cover Art Archive INDEX (json — image METADATA, not + image bytes): the cover picker's one network seam (tests fake exactly + this). Same etiquette as _caa_http_get: throttled, identified, + offline-guarded. Returns the parsed index dict, None when the archive + has no art for the release (404), and raises EnrichTransportError for + anything network-shaped.""" + if not _enrich_network_enabled(): + raise EnrichTransportError("enrichment network disabled") + import requests + _enrich_throttle() + try: + resp = requests.get( + f"https://coverartarchive.org/release/{release_id}", + headers={"User-Agent": _enrich_user_agent(), + "Accept": "application/json"}, + timeout=15, allow_redirects=True) + if resp.status_code == 404: + return None + if resp.status_code != 200: + raise EnrichTransportError(f"cover art archive HTTP {resp.status_code}") + body = resp.json() + return body if isinstance(body, dict) else None + except requests.RequestException as e: + raise EnrichTransportError(str(e)) from e + except ValueError as e: + # Non-JSON body — treat as a transport blip (nothing gets cached, a + # later picker-open retries) rather than caching an empty index. + raise EnrichTransportError(f"cover art archive returned non-JSON: {e}") from e + + +# Per-release lock so two concurrent /art/candidates opens for the SAME +# release serialise their read→fetch→write (the "index cached, no second +# fetch" invariant). Different releases still fetch in parallel; the guard +# lock only protects the tiny registry lookup. +_caa_index_locks: dict[str, threading.Lock] = {} + + +_caa_index_locks_guard = threading.Lock() + + +def _caa_index_lock(release_id: str) -> threading.Lock: + with _caa_index_locks_guard: + lock = _caa_index_locks.get(release_id) + if lock is None: + lock = _caa_index_locks[release_id] = threading.Lock() + return lock + + +def _caa_index_cached(release_id: str) -> list[dict]: + """A release's CAA index images through a TTL-less on-disk cache + (`caa_index_{id}.json` beside the cover files — indexes are stable, and + a 404 is cached as an empty index so a coverless release is never + re-asked). Outside the network seam on purpose: tests fake + _caa_release_index and still exercise this cache. Raises + EnrichTransportError on a cache-miss network failure (the caller stops + asking for further releases); malformed ids/bodies yield [].""" + if not _CAA_ID_RE.match(str(release_id or "")): + return [] + cache_file = _enrichment_art_dir() / f"caa_index_{release_id}.json" + # Hold the per-id lock across the check→fetch→write so a concurrent open + # for the same release finds the freshly-written cache instead of racing a + # second fetch. (The network fetch sleeps in _enrich_throttle under a + # different lock — no deadlock; a different release is never blocked.) + with _caa_index_lock(str(release_id)): + if cache_file.is_file(): + try: + body = json.loads(cache_file.read_text(encoding="utf-8")) + imgs = body.get("images") if isinstance(body, dict) else None + if isinstance(imgs, list): + return imgs + except (OSError, ValueError): + pass # unreadable/corrupt cache → refetch below + body = _caa_release_index(release_id) + if body is None or not isinstance(body.get("images"), list): + body = {"images": []} + try: + cache_file.write_text(json.dumps(body), encoding="utf-8") + except OSError: + pass # cache is best-effort; the response still serves + return body["images"] + + +def _prune_caa_cache() -> None: + """Keep the CAA side of the art cache under its size cap: evict the + oldest caa_* files (mtime LRU) and reset the enrichment rows that pointed + at them. User-override files are never touched.""" + try: + files = sorted(appstate.art_cache_dir.glob("caa_*.jpg"), key=lambda p: p.stat().st_mtime) + total = sum(p.stat().st_size for p in files) + evicted: list[str] = [] + while files and total > _CAA_CACHE_CAP_BYTES: + victim = files.pop(0) + try: + total -= victim.stat().st_size + victim.unlink() + evicted.append(str(victim)) + except OSError: + break + if evicted: + appstate.meta_db.clear_enrichment_art_paths(evicted) + log.info("art cache: evicted %d cover(s) to stay under the cap", len(evicted)) + except Exception: + log.exception("art cache prune failed") + + +def _enrich_art_one(row: dict) -> bool: + """Resolve one matched song's cover-art situation (art worker, phase 3). + Returns True when a cover was actually fetched. Every outcome writes an + art_state so the row never re-queues: + 'pack' — the song ships its own art (it wins; nothing to do) + 'user' — an override exists (it wins; nothing to do) + 'caa' — front cover cached (possibly deduped from an earlier fetch + of the same release — no network on that path) + 'none' — the Cover Art Archive has no cover for this release + Network errors raise EnrichTransportError → the pass pauses and the row + stays unevaluated for the next kick.""" + fn, release_id = row["filename"], row["mb_release_id"] + if not release_id or not _CAA_ID_RE.match(str(release_id)): + # Malformed release id — never build a cache path from it. Settle the + # row as 'error' so it isn't re-queued every pass. + appstate.meta_db.set_enrichment_art(fn, None, "error") + return False + if appstate.song_pack_art_exists(fn): + appstate.meta_db.set_enrichment_art(fn, None, "pack") + return False + if appstate.art_override_paths(fn): + appstate.meta_db.set_enrichment_art(fn, None, "user") + return False + cache_file = _enrichment_art_dir() / f"caa_{release_id}.jpg" + if cache_file.is_file(): + appstate.meta_db.set_enrichment_art(fn, str(cache_file), "caa") + return False + data = _caa_http_get(release_id) + if data is None: + appstate.meta_db.set_enrichment_art(fn, None, "none") + return False + cache_file.write_bytes(data) + appstate.meta_db.set_enrichment_art(fn, str(cache_file), "caa") + _prune_caa_cache() + return True + + +_ENRICH_APPLY_FIELDS = { + # Per-field auto-apply toggle → the candidate fields it governs. The + # MusicBrainz ids + isrc are deliberately NOT here: they're identity, + # not display — the art fetch and any future re-match need them stamped + # even when every display field is toggled off. + "enrich_apply_names": ("artist", "title", "album", "artist_sort"), + "enrich_apply_year": ("year",), + "enrich_apply_genres": ("genres",), +} + + +def _enrich_blocked_apply_keys(cfg: dict) -> frozenset: + """The per-field auto-apply toggle keys that are currently OFF (suppressed). + Its complement (`_ENRICH_APPLY_FIELDS` minus these) is what an automatic + match may canonicalize.""" + return frozenset(k for k in _ENRICH_APPLY_FIELDS if cfg.get(k, True) is False) + + +def _enrich_apply_mask(cfg: dict) -> str: + """Canonical marker of the suppressed apply keys, persisted on each + automatic match so re-enabling a field re-queues the row for backfill + (enrichment_pending) and a partial match can't seed siblings + (enrichment_cache_lookup). '' = nothing suppressed (the default).""" + return ",".join(sorted(_enrich_blocked_apply_keys(cfg))) + + +def _enrich_field_filter(cfg: dict): + """Build the cand filter for AUTOMATIC matches from the per-field + auto-apply settings: strips the display fields whose toggle is off + before they're stamped as canonical. Returns None when everything is + on (the default) so the common path stays zero-copy. Review candidates + and user-confirmed picks bypass this — a match the user confirms in + the modal applies in full.""" + blocked = {f for key in _enrich_blocked_apply_keys(cfg) + for f in _ENRICH_APPLY_FIELDS[key]} + if not blocked: + return None + return lambda cand: {k: v for k, v in cand.items() if k not in blocked} + + +# A per-song LOCK (Fix-metadata popup) → the candidate display keys it +# suppresses on an AUTOMATIC match. Identity keys (recording/release/artist ids, +# isrc) are deliberately absent: a locked DISPLAY field still gets matched for +# art + future re-match, it just isn't re-canonicalized behind the user's back. +_LOCK_FIELD_TO_CAND = { + "artist": ("artist", "artist_sort"), + "title": ("title",), + "album": ("album",), + "year": ("year",), + "genre": ("genres",), +} + + +def _compose_lock_filter(base_filter, locked_fields): + """Wrap the pass's global per-field apply-filter with a per-song filter that + also strips the song's LOCKED display fields, so an automatic match never + re-canonicalizes a field the user pinned. Returns base_filter unchanged when + the song has no relevant lock (the common path).""" + blocked = {ck for f in locked_fields for ck in _LOCK_FIELD_TO_CAND.get(f, ())} + if not blocked: + return base_filter + + def lock_filter(cand): + c = base_filter(cand) if base_filter else cand + return {k: v for k, v in c.items() if k not in blocked} + return lock_filter + + +def _enrich_one(row: dict, auto_min: float | None = None, field_filter=None, + apply_mask: str = "") -> None: + """The matcher (P8; replaces P7's no-op). Precedence per design §5: + + 1. local match-cache by content_hash — another chart of the same + recording already matched/pinned → copy it, NO network; + 2. manifest `mbid` (tier 0) / `isrc` (tier 1) exact keys → direct + lookup, auto; + 3. text search → scored tiers: auto (high) / review (medium — a human + confirms before anything canonicalizes) / failed (low, retried on + backoff). + + `auto_min` is the user's auto-apply confidence setting (None → the + engine default); it moves only the auto/review boundary of step 3 — + the per-field floors and exact-key tiers are unaffected. `field_filter` + (from _enrich_field_filter) strips per-field-disabled display values + from every AUTOMATIC stamp — all three steps here are automatic, so it + applies to each; the review tier stores candidates unfiltered because + accepting one is a user action. Never touches a `manual` row (the + writer enforces it). `apply_mask` (the suppressed keys, from + _enrich_apply_mask) is stamped on each AUTOMATIC match so a later + re-enable re-queues the row for backfill and a partial match can't seed + siblings. Network errors raise EnrichTransportError so the pass pauses + instead of burning attempts while offline.""" + fn, chash = row["filename"], row["content_hash"] + # Respect per-song field LOCKS (Fix-metadata popup): an automatic match must + # not re-canonicalize a field the user pinned. Compose the lock filter onto + # the pass's global apply-filter — both the cache-copy and text-match auto + # paths run their candidate through it. (Review/manual picks bypass the + # filter, so confirming a match in the modal is an explicit override.) + locked = appstate.meta_db.locked_fields(fn) + if locked: + field_filter = _compose_lock_filter(field_filter, locked) + + cached = appstate.meta_db.enrichment_cache_lookup(chash, exclude_filename=fn) + if cached: + score = cached.pop("score", None) + if field_filter: + cached = field_filter(cached) + appstate.meta_db.apply_enrichment_match(fn, chash, "matched", source="cache", + score=score, cand=cached, apply_mask=apply_mask) + return + + ids = _manifest_exact_ids(fn) + if ids.get("mbid"): + cand = _mb_lookup_recording(ids["mbid"]) + if cand: + appstate.meta_db.apply_enrichment_match(fn, chash, "matched", source="mbid", + score=1.0, apply_mask=apply_mask, + cand=field_filter(cand) if field_filter else cand) + return + # A 404'd mbid (typo'd manifest) falls through to the text tiers. + # A pack that left `artist` blank can't be text-matched (search needs an + # artist, and the per-field floor rejects a blank one) — so when it's blank, + # seed the query/scoring from the filename's Artist_Song convention. Seed + # only: fn/chash and the stored row are untouched, and the DISPLAYED values + # still come from the confirmed match. The exact-key tiers above don't need + # it (mbid/isrc identify without text). + ref = row + if not (row.get("artist") or "").strip(): + derived = _artist_title_from_filename(fn) + if derived: + ref = {**row, **derived} + + if ids.get("isrc"): + cands = mb_match.rank_candidates(ref, _mb_lookup_isrc(ids["isrc"])) + if cands: + appstate.meta_db.apply_enrichment_match(fn, chash, "matched", source="isrc", + score=1.0, apply_mask=apply_mask, + cand=field_filter(cands[0]) if field_filter else cands[0]) + return + + cands = _mb_search_recordings(ref.get("artist"), ref.get("title")) + # Alias-enrich promising near-misses (title agrees, primary artist doesn't) + # so a non-Latin-primary artist can confirm via its romanized alias, then + # rank once with the aliases in hand. `ref` carries any filename-derived + # artist seed, so alias scoring runs against the searched identity. + _alias_enrich(ref, cands) + ranked = mb_match.rank_candidates(ref, cands) + best = ranked[0] if ranked else None + tier = mb_match.classify(ref, best, best["score"], auto_min=auto_min) if best else "none" + if tier == "auto": + appstate.meta_db.apply_enrichment_match(fn, chash, "matched", source="text", + score=best["score"], apply_mask=apply_mask, + cand=field_filter(best) if field_filter else best) + elif tier == "review": + appstate.meta_db.apply_enrichment_match(fn, chash, "review", source="text", + score=best["score"], + candidates=ranked[:_ENRICH_MAX_CANDIDATES]) + else: + appstate.meta_db.apply_enrichment_match(fn, chash, "failed", source="text", + score=(best["score"] if best else None), + candidates=ranked[:_ENRICH_MAX_CANDIDATES] or None, + bump_attempts=True) + + +def _background_enrich(): + """One bounded pass, two phases. Phase 1 stamps/refreshes identity-hash + stubs for every song whose identity is new or changed — pure-local, so + hashes stay fresh (and stale matches drop back to `unscanned`) even + fully offline. Phase 2 runs the matcher over those rows plus any + `failed` rows whose backoff has elapsed; a transport failure pauses it + (state untouched, no attempt burned) and the next kick retries. Offline + (kill-switch or the test env) skips phase 2 entirely. Never drains in a + loop — a dead network would make that spin forever. Between songs it + honours the Stop button's cancel flag (phases 2 and 3), so a long trickle + can be halted without waiting for the whole queue to drain.""" + _enrich_status["processed"] = 0 + _enrich_status["total"] = 0 + _enrich_status["matched"] = 0 + _enrich_status["current"] = None + # User settings gate the BACKGROUND matcher only (the review modal's + # manual search/fix stays available when it's off); read once per pass, + # up front so the pending query can honour the per-field apply mask + # (a re-enabled field re-queues its `matched` rows for backfill). + cfg = _load_config(appstate.config_dir / "config.json") or {} + allowed_keys = frozenset(_ENRICH_APPLY_FIELDS) - _enrich_blocked_apply_keys(cfg) + apply_mask = _enrich_apply_mask(cfg) + try: + pending = appstate.meta_db.enrichment_pending(limit=100000, allowed_keys=allowed_keys) + except Exception: + log.exception("enrichment: pending query failed") + return + for row in pending: + try: + appstate.meta_db.upsert_enrichment_stub(row["filename"], row["content_hash"]) + except Exception as e: + log.warning("enrichment stub failed for %s: %s", row.get("filename"), e) + _enrich_status["processed"] += 1 + _enrich_status["last_pass_at"] = time.time() + + if cfg.get("enrich_enabled", True) is False: + if pending: + log.info("Enrichment pass: %d rows stamped (matching disabled in Settings)", len(pending)) + return + try: + auto_min = float(cfg.get("enrich_auto_threshold", 0.9)) + except (TypeError, ValueError): + auto_min = 0.9 + + if not _enrich_network_enabled(): + if pending: + log.info("Enrichment pass: %d rows stamped (network disabled — matching skipped)", len(pending)) + return + + # Scraper options (R1), read from the same per-pass cfg: `mb_on` gates + # the matcher (phase 2), `art_on` the cover-art fetch (phase 3 — the + # Cover Art Archive is the only automatic art source today, so the + # source toggle and the cover-art apply toggle both have to be on). + mb_on = cfg.get("enrich_src_musicbrainz", True) is not False + art_on = (cfg.get("enrich_src_caa", True) is not False + and cfg.get("enrich_apply_art", True) is not False) + field_filter = _enrich_field_filter(cfg) + + now = time.time() + retriable = [] + if mb_on: + try: + retriable = [r for r in appstate.meta_db.enrichment_failed_rows(limit=100000) + if _enrich_backoff_elapsed(r.get("attempts"), r.get("last_attempt_at"), now)] + except Exception: + log.exception("enrichment: failed-row query failed") + elif pending: + log.info("Enrichment pass: %d rows stamped (MusicBrainz source disabled in Settings)", len(pending)) + matched = 0 + # A `failed` row with a changed identity hash can surface in BOTH lists; + # de-dup by filename so each row consumes the rate budget only once. + seen_filenames = set() + queue = [] + for row in (pending + retriable) if mb_on else []: + fn = row.get("filename") + if fn in seen_filenames: + continue + seen_filenames.add(fn) + queue.append(row) + _enrich_status["total"] = len(queue) + for row in queue: + if _enrich_cancel.is_set(): + log.info("enrichment: pass cancelled by user after %d matched", matched) + break + _enrich_status["current"] = row.get("filename") + try: + _enrich_one(row, auto_min=auto_min, field_filter=field_filter, + apply_mask=apply_mask) + matched += 1 + _enrich_status["matched"] = matched + except EnrichTransportError as e: + log.info("enrichment: network unavailable, pass paused (%s)", e) + break + except Exception as e: + log.warning("enrichment failed for %s: %s", row.get("filename"), e) + try: + # Park the row on the failure backoff instead of retrying a + # poisoned input every pass. + appstate.meta_db.apply_enrichment_match( + row["filename"], row["content_hash"], "failed", + source="error", bump_attempts=True) + except Exception: + pass + _enrich_status["current"] = None + if mb_on and (pending or retriable): + log.info("Enrichment pass: %d rows stamped, %d matched", len(pending), matched) + + # Phase 3 — cover art (R3/P9). For freshly-matched songs, resolve the art + # situation once: songs with their own pack art (or a user override) are + # marked and skipped; the rest fetch the release's front cover from the + # Cover Art Archive into the size-capped cache. Same pause-on-transport- + # error rule as matching — a dead network never burns a row's evaluation. + # Rows skipped here stay art_state NULL, so re-enabling the toggles picks + # them up on the next pass — nothing is permanently forfeited. + if not art_on: + return + try: + art_rows = appstate.meta_db.enrichment_art_pending(limit=100000) + except Exception: + log.exception("enrichment: art-pending query failed") + return + fetched = 0 + for row in art_rows: + if _enrich_cancel.is_set(): + log.info("enrichment: art pass cancelled by user after %d fetched", fetched) + break + try: + fetched += 1 if _enrich_art_one(row) else 0 + except EnrichTransportError as e: + log.info("enrichment: network unavailable, art pass paused (%s)", e) + break + except Exception as e: + log.warning("enrichment art failed for %s: %s", row.get("filename"), e) + try: + appstate.meta_db.set_enrichment_art(row["filename"], None, "error") + except Exception: + pass + if art_rows: + log.info("Enrichment art pass: %d evaluated, %d covers fetched", len(art_rows), fetched) + + +def _kick_enrich() -> bool: + """Request an enrichment pass, single-flight + coalescing (the _kick_scan + contract): True = a worker thread was started, False = one is running and + a follow-up pass was queued.""" + global _enrich_pending_pass, _enrich_thread + with _enrich_kick_lock: + if _enrich_status["running"]: + _enrich_pending_pass = True + return False + # A fresh pass supersedes any prior Stop — clear the flag so the new + # pass isn't cancelled the instant it checks (a stale set() from a + # cancelled-then-re-kicked run would otherwise abort it immediately). + _enrich_cancel.clear() + _enrich_status["running"] = True + _enrich_thread = threading.Thread(target=_enrich_runner, daemon=True) + _enrich_thread.start() + return True + + +def _enrich_runner(): + global _enrich_pending_pass + while True: + try: + _background_enrich() + except Exception: + log.exception("background enrichment failed unexpectedly") + with _enrich_kick_lock: + _enrich_status["current"] = None + if _enrich_cancel.is_set(): + # Stop: abandon any coalesced follow-up and clear the flag so the + # next kick starts clean. The current pass already broke out of + # its loop between songs (see _background_enrich). + _enrich_pending_pass = False + _enrich_cancel.clear() + _enrich_status["running"] = False + return + if not _enrich_pending_pass: + _enrich_status["running"] = False + return + _enrich_pending_pass = False diff --git a/server.py b/server.py index 513666a..96253e6 100644 --- a/server.py +++ b/server.py @@ -39,7 +39,6 @@ from tunings import ( # are re-imported because callers outside the DB layer still use them. from metadata_db import ( MetadataDB, - _artist_title_from_filename, _as_int, _effective_keyset_sort, _sqlite_file_integrity_ok, @@ -58,13 +57,13 @@ import appstate # Extracted route modules. They import `appstate`, never `server` — one-way graph. from routers import audio_effects, artist_aliases, loops, playlists, ws_highway, chart, wanted, library_extras, shop, progression, profile, stats, version, diagnostics from routers import tunings as tunings_router +import enrichment import sloppak as sloppak_mod import loosefolder as loosefolder_mod # Pure text-matching engine for MusicBrainz enrichment (P8): denoise/score/ # tier classification + response parsing. No network/DB in there — the # throttled transport and the song_enrichment writes live in this module. import mb_match -import acoustid_match # Metadata extraction lives in a side-effect-free module so ProcessPool # scan workers can import + unpickle _scan_one without re-running this # module's import-time side effects (see lib/scan_worker.py). @@ -1658,7 +1657,6 @@ _scan_rescan_pending = False # use-after-free that segfaults the process (seen flaky in CI). Set by # _kick_scan / _kick_enrich; joined by _join_background_db_threads(). _scan_thread: threading.Thread | None = None -_enrich_thread: threading.Thread | None = None def _join_background_db_threads(timeout: float = 30.0) -> None: @@ -1669,7 +1667,7 @@ def _join_background_db_threads(timeout: float = 30.0) -> None: st = _scan_thread if st is not None and st.is_alive(): st.join(timeout) - et = _enrich_thread + et = enrichment._enrich_thread if et is not None and et.is_alive(): et.join(timeout) @@ -1717,7 +1715,7 @@ def _scan_runner(): # enrichment is a SEPARATE post-scan pass — non-blocking, the library is # usable immediately. The 5-minute periodic rescan re-kicks it, which is # the natural low-priority retry hook. - _kick_enrich() + enrichment._kick_enrich() # ── Metadata enrichment worker (P7 plumbing + P8 matcher) ───────────────────── @@ -1729,631 +1727,77 @@ def _scan_runner(): # lib/mb_match.py. Wrong-match is worse than slow (design §5): medium # confidence goes to the Match-Review queue, never straight to canonical. -_enrich_kick_lock = threading.Lock() -_enrich_pending_pass = False -# processed = phase-1 stubs stamped this pass (legacy field). total/matched = -# the phase-2 MATCHING progress the "Refresh Metadata" batch bar reads (the -# slow, rate-limited part worth a progress readout); current = the song being -# matched right now, which drives the per-tile "working" badge. -_enrich_status = {"running": False, "processed": 0, "last_pass_at": None, - "total": 0, "matched": 0, "current": None} -# Cooperative cancel for the Stop button: the matching/art loops check it -# between songs (an in-flight ≤1/s lookup can't be interrupted, but no new one -# is started). Set by /api/enrichment/cancel, cleared when a fresh pass kicks. -_enrich_cancel = threading.Event() -# Minimum spacing between EXTERNAL lookups (design: ≤1 req/s + local cache). -_ENRICH_MIN_INTERVAL = 1.1 -_enrich_last_fetch = 0.0 -# Serializes throttling across the background daemon thread AND the sync -# /api/enrichment/search route (FastAPI runs sync routes in a threadpool). -_enrich_throttle_lock = threading.Lock() -def _enrichment_art_dir() -> Path: - """The size-capped art cache dir (populated by the Cover Art slice; the - LRU cap policy lands with it). Under CONFIG_DIR so Settings backup/restore - and the docker volume already cover it.""" - d = CONFIG_DIR / "art_cache" - d.mkdir(parents=True, exist_ok=True) - return d -def _enrich_throttle(): - """Block until an external lookup is allowed. Matchers MUST call this - before every network request — and must NOT hold meta_db._lock across the - request (fetch outside the lock, write inside).""" - global _enrich_last_fetch - # Hold the lock across the read, sleep, and write so concurrent callers - # serialize instead of all reading the same stale timestamp and firing - # together (which would burst past MusicBrainz's 1 req/s limit). - with _enrich_throttle_lock: - wait = _ENRICH_MIN_INTERVAL - (time.monotonic() - _enrich_last_fetch) - if wait > 0: - time.sleep(wait) - _enrich_last_fetch = time.monotonic() -class EnrichTransportError(Exception): - """Network-level enrichment failure — offline, DNS, MusicBrainz down or - rate-limiting. Pauses the current pass (rows keep their state and no - attempt is consumed); the next kick (scan-complete / the 5-min periodic - rescan) retries naturally.""" -_MB_API_ROOT = "https://musicbrainz.org/ws/2" -_enrich_ua_cache: str | None = None -def _enrich_user_agent() -> str: - """MusicBrainz etiquette requires a real identifying User-Agent - (app/version + contact URL); anonymous defaults get throttled/blocked.""" - global _enrich_ua_cache - if _enrich_ua_cache is None: - version = "unknown" - try: - vf = Path(__file__).parent / "VERSION" - if vf.exists(): - version = vf.read_text().strip() or "unknown" - except (OSError, UnicodeDecodeError): - pass - _enrich_ua_cache = f"feedBack/{version} (https://github.com/got-feedback/feedBack)" - return _enrich_ua_cache -def _enrich_network_enabled() -> bool: - """False = the matcher runs local-only (hash stamping, cache copies) and - never opens a socket. FEEDBACK_ENRICH_OFFLINE is the explicit user - kill-switch (privacy / air-gapped installs); FEEDBACK_SKIP_STARTUP_TASKS - marks the test/CI environment, where pytest must never reach the network - no matter what a test triggers.""" - return not (_env_flag("FEEDBACK_ENRICH_OFFLINE") - or _env_flag("FEEDBACK_SKIP_STARTUP_TASKS")) -def _mb_http_get(path: str, params: dict) -> dict | None: - """The ONE place enrichment touches the network (tests fake exactly this - seam). Throttled (≤1 req/s via _enrich_throttle), identified (real - User-Agent), offline-guarded. Returns the parsed JSON body, or None for - a 404 lookup; raises EnrichTransportError for anything network-shaped. - NEVER call this while holding meta_db._lock — fetch outside, write - inside.""" - if not _enrich_network_enabled(): - raise EnrichTransportError("enrichment network disabled") - import requests # declared in requirements.txt; lazy so tests never need it - _enrich_throttle() - try: - resp = requests.get( - f"{_MB_API_ROOT}/{path.lstrip('/')}", - params={**params, "fmt": "json"}, - headers={"User-Agent": _enrich_user_agent()}, - timeout=10, - ) - except requests.RequestException as e: - raise EnrichTransportError(str(e)) from e - if resp.status_code == 404: - return None - if resp.status_code == 503: - # MusicBrainz signals rate-limit pressure with 503 — back the whole - # pass off rather than hammering on. - raise EnrichTransportError("musicbrainz 503 (rate limited)") - if resp.status_code != 200: - raise EnrichTransportError(f"musicbrainz HTTP {resp.status_code}") - try: - return resp.json() - except ValueError as e: - raise EnrichTransportError("bad JSON from musicbrainz") from e -def _mb_search_recordings(artist, title, limit: int = 12) -> list[dict]: - """Text search (tier 2–4): denoised Lucene query over /recording. The strict - query drops live-only recordings and the ranker rewards the studio take, so a - slightly larger default result set gives the re-ranker room to surface the - canonical version. - - Runs the strict field-phrase query first (high precision); if it finds - nothing, retries ONCE with a loose term query. The strict phrase only matches - MusicBrainz's *primary* artist/title, so a recording stored under a non-Latin - primary name (大橋純子) whose romanized form ("Junko Ohashi") is only an alias - is invisible to it — the loose query searches aliases and rescues it. The - retry spends a second throttled request only on a miss; results are re-scored - by rank_candidates, so the looser recall doesn't lower match quality - (auto-accept still needs the per-field floors).""" - query = mb_match.build_recording_query(artist, title) - cands: list[dict] = [] - if query: - body = _mb_http_get("recording", {"query": query, "limit": limit}) - cands = mb_match.parse_search_response(body or {}) - if not cands: - loose = mb_match.build_recording_query(artist, title, loose=True) - if loose and loose != query: - body = _mb_http_get("recording", {"query": loose, "limit": limit}) - cands = mb_match.parse_search_response(body or {}) - return cands -def _mb_search_release_groups(query: str, limit: int = 8) -> list[dict]: - """Text search /release-group for the Change-cover picker: albums matching a - free query, each mapped to its Cover Art Archive front thumb. One request; - tiles whose CAA art is missing self-hide client-side (front-250 404s). Lets a - cover be found even for a song with no metadata match (the city-pop pile).""" - q = (query or "").strip() - if not q: - return [] - body = _mb_http_get("release-group", {"query": q, "limit": limit}) - out: list[dict] = [] - for rg in ((body or {}).get("release-groups") or []): - rid = rg.get("id") - if not rid: - continue - # artist-credit is a list of {name, joinphrase, artist} (joinphrase glues - # collaborations) — reconstruct the credited name. - artist = "".join( - (c.get("name", "") + c.get("joinphrase", "")) if isinstance(c, dict) else str(c) - for c in (rg.get("artist-credit") or []) - ).strip() - title = rg.get("title") or "" - year = (rg.get("first-release-date") or "")[:4] - out.append({ - "id": rid, - "label": " · ".join(x for x in (title, artist, year) if x) or title or "Cover", - "thumb_url": f"https://coverartarchive.org/release-group/{rid}/front-250", - }) - return out # ── AcoustID audio fingerprinting (content-based identification) ────────────── # Optional path: requires the Chromaprint `fpcalc` binary AND an AcoustID API # key ($ACOUSTID_API_KEY). Both absent ⇒ graceful no-op; the text matcher runs. -_ACOUSTID_MAX_UPLOAD_BYTES = 256 * 1024 * 1024 # 256 MB — an uncompressed master -def _fpcalc_bin() -> str | None: - """Locate the Chromaprint `fpcalc` binary: $FPCALC override, else PATH.""" - import shutil - cand = os.environ.get("FPCALC") - if cand and Path(cand).exists(): - return cand - return shutil.which("fpcalc") -def _acoustid_settings() -> "tuple[bool, str]": - """(enabled, api_key) for AcoustID, resolved from settings with an env-var - fallback for the key. Opt-in: `acoustid_enabled` defaults off. The key lives - in settings so a user can set it themselves in the UI; $ACOUSTID_API_KEY is a - server-wide fallback for a headless deploy.""" - cfg = _load_config(CONFIG_DIR / "config.json") or {} - enabled = cfg.get("acoustid_enabled", False) is True - key = cfg.get("acoustid_api_key") - if not isinstance(key, str) or not key.strip(): - key = os.environ.get("ACOUSTID_API_KEY", "") - return enabled, (key or "").strip() -def _acoustid_available() -> bool: - """True only when the user opted in, a key is set (settings or env), the - network is on, AND fpcalc exists.""" - enabled, key = _acoustid_settings() - return (enabled - and _enrich_network_enabled() - and acoustid_match.is_configured(key) - and _fpcalc_bin() is not None) -def _fpcalc(path: str) -> "tuple[int, str] | None": - """Fingerprint a local audio file → (duration_seconds, fingerprint). None on - any failure (missing binary/file, decode error, timeout).""" - binp = _fpcalc_bin() - if not binp or not Path(path).exists(): - return None - import subprocess - import json as _json - try: - pr = subprocess.run([binp, "-json", str(path)], - capture_output=True, timeout=30) - except Exception: - return None - if pr.returncode != 0: - return None - try: - data = _json.loads(pr.stdout.decode("utf-8", "replace")) - dur = int(round(float(data.get("duration")))) - fp = str(data.get("fingerprint") or "") - except Exception: - return None - if not fp or dur <= 0: - return None - return dur, fp -def _acoustid_lookup(duration: int, fingerprint: str) -> list[dict]: - """Look a fingerprint up on AcoustID → candidate dicts (mb_match shape). - Throttled + offline-guarded like the MusicBrainz path. [] when unavailable - or no hit; raises EnrichTransportError for network-shaped failures.""" - _, key = _acoustid_settings() - if not key or not _enrich_network_enabled(): - return [] - import requests - _enrich_throttle() - try: - # POST, not GET: a fingerprint is multi-KB (a 3.5-min track is ~3.5k - # chars), so a GET crams it into the URL and a long song overflows the - # server's URL limit → a spurious failure. AcoustID accepts the same - # params form-encoded in the body. - resp = requests.post( - f"{acoustid_match.ACOUSTID_API_ROOT}/lookup", - data={ - "client": key, "format": "json", - "meta": acoustid_match.LOOKUP_META, - "duration": duration, "fingerprint": fingerprint, - }, - headers={"User-Agent": _enrich_user_agent()}, - timeout=10, - ) - except requests.RequestException as e: - raise EnrichTransportError(str(e)) from e - if resp.status_code == 429: - raise EnrichTransportError("acoustid 429 (rate limited)") - if resp.status_code != 200: - raise EnrichTransportError(f"acoustid HTTP {resp.status_code}") - try: - body = resp.json() - except ValueError as e: - raise EnrichTransportError("bad JSON from acoustid") from e - return acoustid_match.parse_lookup_response(body) -def _identify_by_fingerprint(path: str) -> list[dict]: - """fpcalc + AcoustID lookup for a local audio file. [] if fingerprinting is - unavailable, the file can't be read, or nothing matched. Available to the - library-enrichment pipeline as well as the /identify endpoint.""" - if not _acoustid_available(): - return [] - fp = _fpcalc(path) - if not fp: - return [] - return _acoustid_lookup(fp[0], fp[1]) -def _acoustid_gate() -> "JSONResponse | None": - """Shared availability gate for the identify endpoints: None when ready, - else a 412 needs_setup (opt-in off / no key → the UI re-prompts) or a 503 - (set up but fpcalc/network missing). Never lets a caller pretend a - fingerprint ran.""" - if _acoustid_available(): - return None - enabled, key = _acoustid_settings() - if not enabled or not key: - return JSONResponse( - {"error": "audio fingerprinting not set up", "needs_setup": True, - "detail": "Turn on AcoustID and add a free API key to identify by audio — " - "it reads the recording itself, far more reliable than text search."}, - status_code=412) - return JSONResponse( - {"error": "audio fingerprinting unavailable", "needs_setup": False, - "detail": "the fpcalc (Chromaprint) binary was not found on the server"}, - status_code=503) -def _song_audio_file(filename: str) -> "str | None": - """Resolve a LIBRARY song (by filename/id) to a local master-audio file for - fingerprinting: the full-mix `original_audio` extracted from a sloppak, or a - loose folder's audio. None when the song can't be found or ships no full-mix - audio (some packs carry only stems). Mirrors serve_sloppak_file's containment - guards so a crafted filename can't read outside DLC_DIR / the pack.""" - dlc = _get_dlc_dir() - if not dlc: - return None - resolved = _resolve_dlc_path(dlc, filename) - if resolved is None or not resolved.exists(): - return None - if sloppak_mod.is_sloppak(resolved): - try: - canon = resolved.relative_to(dlc.resolve()).as_posix() - except ValueError: - return None - rel = (sloppak_mod.load_manifest(resolved) or {}).get("original_audio") - if not isinstance(rel, str) or not rel.strip(): - return None - src = sloppak_mod.get_cached_source_dir(canon) - if src is None: - try: - src = sloppak_mod.resolve_source_dir(canon, dlc, SLOPPAK_CACHE_DIR) - except Exception: - return None - target = (src / rel.strip()).resolve() - try: - target.relative_to(src.resolve()) - except ValueError: - return None - return str(target) if target.is_file() else None - try: - audio = loosefolder_mod.find_audio(resolved) - except Exception: - audio = None - return str(audio) if audio and Path(str(audio)).is_file() else None -def _mb_lookup_recording(mbid: str) -> dict | None: - """Direct lookup for a manifest-carried recording MBID (tier 0).""" - body = _mb_http_get( - f"recording/{mbid}", - {"inc": "artist-credits+releases+release-groups+isrcs+genres"}) - return mb_match.parse_recording_doc(body) if body else None -def _mb_lookup_isrc(isrc: str) -> list[dict]: - """Recordings registered under a manifest-carried ISRC (tier 1).""" - body = _mb_http_get( - f"isrc/{isrc}", {"inc": "artist-credits+releases+release-groups"}) - if not body: - return [] - docs = body.get("recordings") or [] - return [c for c in (mb_match.parse_recording_doc(d) for d in docs) if c] -# Strict shapes for the manifest's optional identity keys (feedpak spec §5.1). -# Validated before use — the mbid is interpolated into a URL path, so junk or -# hostile manifest values must never reach the request line. -_MBID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") -_ISRC_RE = re.compile(r"^[A-Z]{2}[A-Z0-9]{3}[0-9]{7}$") -# ── Alias-aware scoring ─────────────────────────────────────────────────────── -# MusicBrainz stores many artists under a non-Latin PRIMARY name (大橋純子) with -# the romanized form ("Junko Ohashi") only as an ALIAS. A recording search -# returns the primary name in its artist-credit, never the aliases — so scoring -# a romanized reference against the primary gives 0 and the match can't confirm. -# We fetch the artist's aliases (one throttled lookup, process-cached) and hand -# them to the scorer, but ONLY for a promising near-miss (title already agrees, -# artist doesn't) so a normal pass spends no extra requests. -_ALIAS_ENRICH_MAX = 3 # cap alias lookups per song/search (each is ≤1/s) -_artist_alias_cache: dict[str, list[str]] = {} -def _mb_artist_aliases(artist_id: str) -> list[str]: - """Romanized/alternate names for a MusicBrainz artist, process-cached (an - artist recurs across a whole discography, so a library of one artist costs - ONE lookup). Returns [] for an unknown/aliasless artist. Raises - EnrichTransportError on a network failure so the caller pauses the pass - (nothing is cached on failure → retried next pass).""" - aid = str(artist_id or "") - if aid in _artist_alias_cache: - return _artist_alias_cache[aid] - if not _MBID_RE.match(aid): - return [] - body = _mb_http_get(f"artist/{aid}", {"inc": "aliases"}) - names: list[str] = [] - if body: - sort_name = str(body.get("sort-name") or "").strip() - if sort_name: - names.append(sort_name) # often the romanized form for JP artists - for al in body.get("aliases") or []: - if isinstance(al, dict) and al.get("name"): - names.append(str(al["name"])) - seen: set[str] = set() - out: list[str] = [] - for n in names: - k = n.casefold() - if k and k not in seen: - seen.add(k) - out.append(n) - out = out[:12] - _artist_alias_cache[aid] = out - return out -def _alias_enrich(ref: dict, cands: list[dict]) -> None: - """Attach `artist_aliases` in place to candidates that look like the - non-Latin-primary case — title agrees with the reference but the primary - artist doesn't — so the scorer can confirm them via a romanized alias. - Bounded by _ALIAS_ENRICH_MAX + the process cache; a no-op when the - reference has no artist or nothing is aliasable.""" - ref_artist = (ref.get("artist") or "").strip() - if not ref_artist: - return - spent = 0 - for c in cands: - if spent >= _ALIAS_ENRICH_MAX: - break - if not isinstance(c, dict) or c.get("artist_aliases") is not None: - continue - aid = c.get("artist_id") - if not aid: - continue - # Only spend a lookup on a promising near-miss: the title already - # matches, but the primary artist doesn't (that's the alias signature). - if mb_match.similarity(ref.get("title"), c.get("title")) < mb_match.AUTO_TITLE_MIN: - continue - if mb_match.similarity(ref_artist, c.get("artist"), artist=True) >= mb_match.AUTO_ARTIST_MIN: - continue - c["artist_aliases"] = _mb_artist_aliases(aid) # cached; attach [] to avoid refetch - spent += 1 -def _manifest_exact_ids(filename: str) -> dict: - """Optional `mbid`/`isrc` from the pack manifest — the spec's additive - identity keys. Feature-detected: packs published before that spec - revision simply lack them and fall through to text matching. READ-only: - enrichment never writes anything into pack files.""" - try: - dlc = _get_dlc_dir() - if not dlc: - return {} - p = _resolve_dlc_path(dlc, filename) - if p is None or not p.exists() or not sloppak_mod.is_sloppak(p): - return {} - manifest = sloppak_mod.load_manifest(p) or {} - except Exception: - return {} - out = {} - mbid = str(manifest.get("mbid", "") or "").strip().lower() - if _MBID_RE.match(mbid): - out["mbid"] = mbid - isrc = str(manifest.get("isrc", "") or "").strip().upper() - # Spec 1.14.0: the stored form is the bare 12-char code, but ISRCs - # circulate hyphenated in the wild (US-ABC-24-00001) — the separators - # are presentation, not part of the code, so a hand-authored display - # form still matches (consumers SHOULD strip before comparing). - isrc = isrc.replace("-", "").replace(" ", "") - if _ISRC_RE.match(isrc): - out["isrc"] = isrc - return out -# Failed-row retry backoff: 1 h after the first failed attempt, doubling per -# attempt, capped at a week — a permanently-unmatchable obscure chart must -# not re-hammer MusicBrainz on every scan kick. -_ENRICH_BACKOFF_BASE = 3600.0 -_ENRICH_BACKOFF_CAP = 7 * 86400.0 -def _enrich_backoff_elapsed(attempts, last_attempt_at, now: float) -> bool: - if not last_attempt_at: - return True - delay = min(_ENRICH_BACKOFF_BASE * (2 ** max(0, int(attempts or 1) - 1)), - _ENRICH_BACKOFF_CAP) - return (now - float(last_attempt_at)) >= delay -# Review tier keeps a short ranked candidate list for the drawer; more than a -# handful is noise the user has to scroll past. -_ENRICH_MAX_CANDIDATES = 5 - -# ── Cover art (R3/P9) ───────────────────────────────────────────────────────── -# The art cache dir (CONFIG_DIR/art_cache) holds two kinds of file: -# {safe_name}.png / .gif — USER OVERRIDES (upload or URL-fetch; never -# evicted, removed only with the song or by the -# explicit remove-override route) -# caa_{release_mbid}.jpg — COVER ART ARCHIVE fetches, keyed by release so -# every chart of the same release shares one file; -# size-capped LRU (evictions reset the enrichment -# rows so a later pass may re-fetch) -_CAA_CACHE_CAP_BYTES = 200 * 1024 * 1024 -# Per-cover cap on a single CAA fetch. The 500px thumbnail is normally tens of -# KB; this bounds any one response independently of the aggregate LRU cap so a -# single oversized (or misbehaving) release can't blow up memory/disk. -_CAA_MAX_BYTES = 10 * 1024 * 1024 -# A release MBID is a UUID; before interpolating it into a cache-file path we -# require a conservative token (alphanumerics, hyphen, underscore only) so no -# separator or '.' can ever appear — blocks path traversal. Defence in depth: -# cheap even though the DB only ever holds MusicBrainz UUIDs. (Distinct name -# from the strict recording-MBID _MBID_RE above — this only gates a filename.) -_CAA_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") -def _caa_http_get(release_id: str) -> bytes | None: - """Fetch a release's front cover from the Cover Art Archive — the one - network seam of the art layer (tests fake exactly this). Same etiquette - as the MusicBrainz client: throttled, identified, offline-guarded. - Returns the image bytes, None when the release has no cover (404), and - raises EnrichTransportError for anything network-shaped.""" - if not _enrich_network_enabled(): - raise EnrichTransportError("enrichment network disabled") - import requests - _enrich_throttle() - try: - with requests.get( - f"https://coverartarchive.org/release/{release_id}/front-500", - headers={"User-Agent": _enrich_user_agent()}, - timeout=15, allow_redirects=True, stream=True, - ) as resp: - if resp.status_code == 404: - return None - if resp.status_code != 200: - raise EnrichTransportError(f"cover art archive HTTP {resp.status_code}") - # Stream with a per-file cap so a huge response never fully downloads. - data = b"" - for chunk in resp.iter_content(65536): - data += chunk - if len(data) > _CAA_MAX_BYTES: - # Not network-shaped: settle just this row as 'error' (the - # art loop's generic handler) rather than pausing the pass. - raise ValueError("cover art exceeds size cap") - return data - except requests.RequestException as e: - raise EnrichTransportError(str(e)) from e -def _caa_release_index(release_id: str) -> dict | None: - """Fetch a release's Cover Art Archive INDEX (json — image METADATA, not - image bytes): the cover picker's one network seam (tests fake exactly - this). Same etiquette as _caa_http_get: throttled, identified, - offline-guarded. Returns the parsed index dict, None when the archive - has no art for the release (404), and raises EnrichTransportError for - anything network-shaped.""" - if not _enrich_network_enabled(): - raise EnrichTransportError("enrichment network disabled") - import requests - _enrich_throttle() - try: - resp = requests.get( - f"https://coverartarchive.org/release/{release_id}", - headers={"User-Agent": _enrich_user_agent(), - "Accept": "application/json"}, - timeout=15, allow_redirects=True) - if resp.status_code == 404: - return None - if resp.status_code != 200: - raise EnrichTransportError(f"cover art archive HTTP {resp.status_code}") - body = resp.json() - return body if isinstance(body, dict) else None - except requests.RequestException as e: - raise EnrichTransportError(str(e)) from e - except ValueError as e: - # Non-JSON body — treat as a transport blip (nothing gets cached, a - # later picker-open retries) rather than caching an empty index. - raise EnrichTransportError(f"cover art archive returned non-JSON: {e}") from e -# Per-release lock so two concurrent /art/candidates opens for the SAME -# release serialise their read→fetch→write (the "index cached, no second -# fetch" invariant). Different releases still fetch in parallel; the guard -# lock only protects the tiny registry lookup. -_caa_index_locks: dict[str, threading.Lock] = {} -_caa_index_locks_guard = threading.Lock() -def _caa_index_lock(release_id: str) -> threading.Lock: - with _caa_index_locks_guard: - lock = _caa_index_locks.get(release_id) - if lock is None: - lock = _caa_index_locks[release_id] = threading.Lock() - return lock -def _caa_index_cached(release_id: str) -> list[dict]: - """A release's CAA index images through a TTL-less on-disk cache - (`caa_index_{id}.json` beside the cover files — indexes are stable, and - a 404 is cached as an empty index so a coverless release is never - re-asked). Outside the network seam on purpose: tests fake - _caa_release_index and still exercise this cache. Raises - EnrichTransportError on a cache-miss network failure (the caller stops - asking for further releases); malformed ids/bodies yield [].""" - if not _CAA_ID_RE.match(str(release_id or "")): - return [] - cache_file = _enrichment_art_dir() / f"caa_index_{release_id}.json" - # Hold the per-id lock across the check→fetch→write so a concurrent open - # for the same release finds the freshly-written cache instead of racing a - # second fetch. (The network fetch sleeps in _enrich_throttle under a - # different lock — no deadlock; a different release is never blocked.) - with _caa_index_lock(str(release_id)): - if cache_file.is_file(): - try: - body = json.loads(cache_file.read_text(encoding="utf-8")) - imgs = body.get("images") if isinstance(body, dict) else None - if isinstance(imgs, list): - return imgs - except (OSError, ValueError): - pass # unreadable/corrupt cache → refetch below - body = _caa_release_index(release_id) - if body is None or not isinstance(body.get("images"), list): - body = {"images": []} - try: - cache_file.write_text(json.dumps(body), encoding="utf-8") - except OSError: - pass # cache is best-effort; the response still serves - return body["images"] + def _art_safe_name(filename: str) -> str: @@ -2390,411 +1834,38 @@ def _song_pack_art_exists(filename: str) -> bool: return False -def _prune_caa_cache() -> None: - """Keep the CAA side of the art cache under its size cap: evict the - oldest caa_* files (mtime LRU) and reset the enrichment rows that pointed - at them. User-override files are never touched.""" - try: - files = sorted(ART_CACHE_DIR.glob("caa_*.jpg"), key=lambda p: p.stat().st_mtime) - total = sum(p.stat().st_size for p in files) - evicted: list[str] = [] - while files and total > _CAA_CACHE_CAP_BYTES: - victim = files.pop(0) - try: - total -= victim.stat().st_size - victim.unlink() - evicted.append(str(victim)) - except OSError: - break - if evicted: - meta_db.clear_enrichment_art_paths(evicted) - log.info("art cache: evicted %d cover(s) to stay under the cap", len(evicted)) - except Exception: - log.exception("art cache prune failed") +# Publish the art cache dir + the two shared art helpers to the enrichment seam +# (lib/enrichment.py's worker calls these; the defs stay here because the art / +# delete routes share them). configure() is idempotent/additive. +appstate.configure( + art_cache_dir=ART_CACHE_DIR, + song_pack_art_exists=_song_pack_art_exists, + art_override_paths=_art_override_paths, +) + + -def _enrich_art_one(row: dict) -> bool: - """Resolve one matched song's cover-art situation (art worker, phase 3). - Returns True when a cover was actually fetched. Every outcome writes an - art_state so the row never re-queues: - 'pack' — the song ships its own art (it wins; nothing to do) - 'user' — an override exists (it wins; nothing to do) - 'caa' — front cover cached (possibly deduped from an earlier fetch - of the same release — no network on that path) - 'none' — the Cover Art Archive has no cover for this release - Network errors raise EnrichTransportError → the pass pauses and the row - stays unevaluated for the next kick.""" - fn, release_id = row["filename"], row["mb_release_id"] - if not release_id or not _CAA_ID_RE.match(str(release_id)): - # Malformed release id — never build a cache path from it. Settle the - # row as 'error' so it isn't re-queued every pass. - meta_db.set_enrichment_art(fn, None, "error") - return False - if _song_pack_art_exists(fn): - meta_db.set_enrichment_art(fn, None, "pack") - return False - if _art_override_paths(fn): - meta_db.set_enrichment_art(fn, None, "user") - return False - cache_file = _enrichment_art_dir() / f"caa_{release_id}.jpg" - if cache_file.is_file(): - meta_db.set_enrichment_art(fn, str(cache_file), "caa") - return False - data = _caa_http_get(release_id) - if data is None: - meta_db.set_enrichment_art(fn, None, "none") - return False - cache_file.write_bytes(data) - meta_db.set_enrichment_art(fn, str(cache_file), "caa") - _prune_caa_cache() - return True -_ENRICH_APPLY_FIELDS = { - # Per-field auto-apply toggle → the candidate fields it governs. The - # MusicBrainz ids + isrc are deliberately NOT here: they're identity, - # not display — the art fetch and any future re-match need them stamped - # even when every display field is toggled off. - "enrich_apply_names": ("artist", "title", "album", "artist_sort"), - "enrich_apply_year": ("year",), - "enrich_apply_genres": ("genres",), -} -def _enrich_blocked_apply_keys(cfg: dict) -> frozenset: - """The per-field auto-apply toggle keys that are currently OFF (suppressed). - Its complement (`_ENRICH_APPLY_FIELDS` minus these) is what an automatic - match may canonicalize.""" - return frozenset(k for k in _ENRICH_APPLY_FIELDS if cfg.get(k, True) is False) -def _enrich_apply_mask(cfg: dict) -> str: - """Canonical marker of the suppressed apply keys, persisted on each - automatic match so re-enabling a field re-queues the row for backfill - (enrichment_pending) and a partial match can't seed siblings - (enrichment_cache_lookup). '' = nothing suppressed (the default).""" - return ",".join(sorted(_enrich_blocked_apply_keys(cfg))) -def _enrich_field_filter(cfg: dict): - """Build the cand filter for AUTOMATIC matches from the per-field - auto-apply settings: strips the display fields whose toggle is off - before they're stamped as canonical. Returns None when everything is - on (the default) so the common path stays zero-copy. Review candidates - and user-confirmed picks bypass this — a match the user confirms in - the modal applies in full.""" - blocked = {f for key in _enrich_blocked_apply_keys(cfg) - for f in _ENRICH_APPLY_FIELDS[key]} - if not blocked: - return None - return lambda cand: {k: v for k, v in cand.items() if k not in blocked} -# A per-song LOCK (Fix-metadata popup) → the candidate display keys it -# suppresses on an AUTOMATIC match. Identity keys (recording/release/artist ids, -# isrc) are deliberately absent: a locked DISPLAY field still gets matched for -# art + future re-match, it just isn't re-canonicalized behind the user's back. -_LOCK_FIELD_TO_CAND = { - "artist": ("artist", "artist_sort"), - "title": ("title",), - "album": ("album",), - "year": ("year",), - "genre": ("genres",), -} -def _compose_lock_filter(base_filter, locked_fields): - """Wrap the pass's global per-field apply-filter with a per-song filter that - also strips the song's LOCKED display fields, so an automatic match never - re-canonicalizes a field the user pinned. Returns base_filter unchanged when - the song has no relevant lock (the common path).""" - blocked = {ck for f in locked_fields for ck in _LOCK_FIELD_TO_CAND.get(f, ())} - if not blocked: - return base_filter - - def lock_filter(cand): - c = base_filter(cand) if base_filter else cand - return {k: v for k, v in c.items() if k not in blocked} - return lock_filter -def _enrich_one(row: dict, auto_min: float | None = None, field_filter=None, - apply_mask: str = "") -> None: - """The matcher (P8; replaces P7's no-op). Precedence per design §5: - - 1. local match-cache by content_hash — another chart of the same - recording already matched/pinned → copy it, NO network; - 2. manifest `mbid` (tier 0) / `isrc` (tier 1) exact keys → direct - lookup, auto; - 3. text search → scored tiers: auto (high) / review (medium — a human - confirms before anything canonicalizes) / failed (low, retried on - backoff). - - `auto_min` is the user's auto-apply confidence setting (None → the - engine default); it moves only the auto/review boundary of step 3 — - the per-field floors and exact-key tiers are unaffected. `field_filter` - (from _enrich_field_filter) strips per-field-disabled display values - from every AUTOMATIC stamp — all three steps here are automatic, so it - applies to each; the review tier stores candidates unfiltered because - accepting one is a user action. Never touches a `manual` row (the - writer enforces it). `apply_mask` (the suppressed keys, from - _enrich_apply_mask) is stamped on each AUTOMATIC match so a later - re-enable re-queues the row for backfill and a partial match can't seed - siblings. Network errors raise EnrichTransportError so the pass pauses - instead of burning attempts while offline.""" - fn, chash = row["filename"], row["content_hash"] - # Respect per-song field LOCKS (Fix-metadata popup): an automatic match must - # not re-canonicalize a field the user pinned. Compose the lock filter onto - # the pass's global apply-filter — both the cache-copy and text-match auto - # paths run their candidate through it. (Review/manual picks bypass the - # filter, so confirming a match in the modal is an explicit override.) - locked = meta_db.locked_fields(fn) - if locked: - field_filter = _compose_lock_filter(field_filter, locked) - - cached = meta_db.enrichment_cache_lookup(chash, exclude_filename=fn) - if cached: - score = cached.pop("score", None) - if field_filter: - cached = field_filter(cached) - meta_db.apply_enrichment_match(fn, chash, "matched", source="cache", - score=score, cand=cached, apply_mask=apply_mask) - return - - ids = _manifest_exact_ids(fn) - if ids.get("mbid"): - cand = _mb_lookup_recording(ids["mbid"]) - if cand: - meta_db.apply_enrichment_match(fn, chash, "matched", source="mbid", - score=1.0, apply_mask=apply_mask, - cand=field_filter(cand) if field_filter else cand) - return - # A 404'd mbid (typo'd manifest) falls through to the text tiers. - # A pack that left `artist` blank can't be text-matched (search needs an - # artist, and the per-field floor rejects a blank one) — so when it's blank, - # seed the query/scoring from the filename's Artist_Song convention. Seed - # only: fn/chash and the stored row are untouched, and the DISPLAYED values - # still come from the confirmed match. The exact-key tiers above don't need - # it (mbid/isrc identify without text). - ref = row - if not (row.get("artist") or "").strip(): - derived = _artist_title_from_filename(fn) - if derived: - ref = {**row, **derived} - - if ids.get("isrc"): - cands = mb_match.rank_candidates(ref, _mb_lookup_isrc(ids["isrc"])) - if cands: - meta_db.apply_enrichment_match(fn, chash, "matched", source="isrc", - score=1.0, apply_mask=apply_mask, - cand=field_filter(cands[0]) if field_filter else cands[0]) - return - - cands = _mb_search_recordings(ref.get("artist"), ref.get("title")) - # Alias-enrich promising near-misses (title agrees, primary artist doesn't) - # so a non-Latin-primary artist can confirm via its romanized alias, then - # rank once with the aliases in hand. `ref` carries any filename-derived - # artist seed, so alias scoring runs against the searched identity. - _alias_enrich(ref, cands) - ranked = mb_match.rank_candidates(ref, cands) - best = ranked[0] if ranked else None - tier = mb_match.classify(ref, best, best["score"], auto_min=auto_min) if best else "none" - if tier == "auto": - meta_db.apply_enrichment_match(fn, chash, "matched", source="text", - score=best["score"], apply_mask=apply_mask, - cand=field_filter(best) if field_filter else best) - elif tier == "review": - meta_db.apply_enrichment_match(fn, chash, "review", source="text", - score=best["score"], - candidates=ranked[:_ENRICH_MAX_CANDIDATES]) - else: - meta_db.apply_enrichment_match(fn, chash, "failed", source="text", - score=(best["score"] if best else None), - candidates=ranked[:_ENRICH_MAX_CANDIDATES] or None, - bump_attempts=True) -def _background_enrich(): - """One bounded pass, two phases. Phase 1 stamps/refreshes identity-hash - stubs for every song whose identity is new or changed — pure-local, so - hashes stay fresh (and stale matches drop back to `unscanned`) even - fully offline. Phase 2 runs the matcher over those rows plus any - `failed` rows whose backoff has elapsed; a transport failure pauses it - (state untouched, no attempt burned) and the next kick retries. Offline - (kill-switch or the test env) skips phase 2 entirely. Never drains in a - loop — a dead network would make that spin forever. Between songs it - honours the Stop button's cancel flag (phases 2 and 3), so a long trickle - can be halted without waiting for the whole queue to drain.""" - _enrich_status["processed"] = 0 - _enrich_status["total"] = 0 - _enrich_status["matched"] = 0 - _enrich_status["current"] = None - # User settings gate the BACKGROUND matcher only (the review modal's - # manual search/fix stays available when it's off); read once per pass, - # up front so the pending query can honour the per-field apply mask - # (a re-enabled field re-queues its `matched` rows for backfill). - cfg = _load_config(CONFIG_DIR / "config.json") or {} - allowed_keys = frozenset(_ENRICH_APPLY_FIELDS) - _enrich_blocked_apply_keys(cfg) - apply_mask = _enrich_apply_mask(cfg) - try: - pending = meta_db.enrichment_pending(limit=100000, allowed_keys=allowed_keys) - except Exception: - log.exception("enrichment: pending query failed") - return - for row in pending: - try: - meta_db.upsert_enrichment_stub(row["filename"], row["content_hash"]) - except Exception as e: - log.warning("enrichment stub failed for %s: %s", row.get("filename"), e) - _enrich_status["processed"] += 1 - _enrich_status["last_pass_at"] = time.time() - - if cfg.get("enrich_enabled", True) is False: - if pending: - log.info("Enrichment pass: %d rows stamped (matching disabled in Settings)", len(pending)) - return - try: - auto_min = float(cfg.get("enrich_auto_threshold", 0.9)) - except (TypeError, ValueError): - auto_min = 0.9 - - if not _enrich_network_enabled(): - if pending: - log.info("Enrichment pass: %d rows stamped (network disabled — matching skipped)", len(pending)) - return - - # Scraper options (R1), read from the same per-pass cfg: `mb_on` gates - # the matcher (phase 2), `art_on` the cover-art fetch (phase 3 — the - # Cover Art Archive is the only automatic art source today, so the - # source toggle and the cover-art apply toggle both have to be on). - mb_on = cfg.get("enrich_src_musicbrainz", True) is not False - art_on = (cfg.get("enrich_src_caa", True) is not False - and cfg.get("enrich_apply_art", True) is not False) - field_filter = _enrich_field_filter(cfg) - - now = time.time() - retriable = [] - if mb_on: - try: - retriable = [r for r in meta_db.enrichment_failed_rows(limit=100000) - if _enrich_backoff_elapsed(r.get("attempts"), r.get("last_attempt_at"), now)] - except Exception: - log.exception("enrichment: failed-row query failed") - elif pending: - log.info("Enrichment pass: %d rows stamped (MusicBrainz source disabled in Settings)", len(pending)) - matched = 0 - # A `failed` row with a changed identity hash can surface in BOTH lists; - # de-dup by filename so each row consumes the rate budget only once. - seen_filenames = set() - queue = [] - for row in (pending + retriable) if mb_on else []: - fn = row.get("filename") - if fn in seen_filenames: - continue - seen_filenames.add(fn) - queue.append(row) - _enrich_status["total"] = len(queue) - for row in queue: - if _enrich_cancel.is_set(): - log.info("enrichment: pass cancelled by user after %d matched", matched) - break - _enrich_status["current"] = row.get("filename") - try: - _enrich_one(row, auto_min=auto_min, field_filter=field_filter, - apply_mask=apply_mask) - matched += 1 - _enrich_status["matched"] = matched - except EnrichTransportError as e: - log.info("enrichment: network unavailable, pass paused (%s)", e) - break - except Exception as e: - log.warning("enrichment failed for %s: %s", row.get("filename"), e) - try: - # Park the row on the failure backoff instead of retrying a - # poisoned input every pass. - meta_db.apply_enrichment_match( - row["filename"], row["content_hash"], "failed", - source="error", bump_attempts=True) - except Exception: - pass - _enrich_status["current"] = None - if mb_on and (pending or retriable): - log.info("Enrichment pass: %d rows stamped, %d matched", len(pending), matched) - - # Phase 3 — cover art (R3/P9). For freshly-matched songs, resolve the art - # situation once: songs with their own pack art (or a user override) are - # marked and skipped; the rest fetch the release's front cover from the - # Cover Art Archive into the size-capped cache. Same pause-on-transport- - # error rule as matching — a dead network never burns a row's evaluation. - # Rows skipped here stay art_state NULL, so re-enabling the toggles picks - # them up on the next pass — nothing is permanently forfeited. - if not art_on: - return - try: - art_rows = meta_db.enrichment_art_pending(limit=100000) - except Exception: - log.exception("enrichment: art-pending query failed") - return - fetched = 0 - for row in art_rows: - if _enrich_cancel.is_set(): - log.info("enrichment: art pass cancelled by user after %d fetched", fetched) - break - try: - fetched += 1 if _enrich_art_one(row) else 0 - except EnrichTransportError as e: - log.info("enrichment: network unavailable, art pass paused (%s)", e) - break - except Exception as e: - log.warning("enrichment art failed for %s: %s", row.get("filename"), e) - try: - meta_db.set_enrichment_art(row["filename"], None, "error") - except Exception: - pass - if art_rows: - log.info("Enrichment art pass: %d evaluated, %d covers fetched", len(art_rows), fetched) -def _kick_enrich() -> bool: - """Request an enrichment pass, single-flight + coalescing (the _kick_scan - contract): True = a worker thread was started, False = one is running and - a follow-up pass was queued.""" - global _enrich_pending_pass, _enrich_thread - with _enrich_kick_lock: - if _enrich_status["running"]: - _enrich_pending_pass = True - return False - # A fresh pass supersedes any prior Stop — clear the flag so the new - # pass isn't cancelled the instant it checks (a stale set() from a - # cancelled-then-re-kicked run would otherwise abort it immediately). - _enrich_cancel.clear() - _enrich_status["running"] = True - _enrich_thread = threading.Thread(target=_enrich_runner, daemon=True) - _enrich_thread.start() - return True -def _enrich_runner(): - global _enrich_pending_pass - while True: - try: - _background_enrich() - except Exception: - log.exception("background enrichment failed unexpectedly") - with _enrich_kick_lock: - _enrich_status["current"] = None - if _enrich_cancel.is_set(): - # Stop: abandon any coalesced follow-up and clear the flag so the - # next kick starts clean. The current pass already broke out of - # its loop between songs (see _background_enrich). - _enrich_pending_pass = False - _enrich_cancel.clear() - _enrich_status["running"] = False - return - if not _enrich_pending_pass: - _enrich_status["running"] = False - return - _enrich_pending_pass = False # ── Register plugin API endpoints (lightweight, before app starts) ─────────── @@ -3214,18 +2285,18 @@ def enrichment_status(): Ambient tool-state for the match-review UI (never a home-screen score — design §11); also what tests poke.""" return { - "running": _enrich_status["running"], - "processed": _enrich_status["processed"], - "last_pass_at": _enrich_status["last_pass_at"], + "running": enrichment._enrich_status["running"], + "processed": enrichment._enrich_status["processed"], + "last_pass_at": enrichment._enrich_status["last_pass_at"], "states": meta_db.enrichment_state_counts(), "total_songs": meta_db.count(), # Per-pass matching progress for the "Refresh Metadata" batch bar + # per-tile badges (total = songs queued to match this pass, matched = # done so far, current = the one being matched now). - "total": _enrich_status.get("total", 0), - "matched": _enrich_status.get("matched", 0), - "current": _enrich_status.get("current"), - "cancelling": _enrich_cancel.is_set(), + "total": enrichment._enrich_status.get("total", 0), + "matched": enrichment._enrich_status.get("matched", 0), + "current": enrichment._enrich_status.get("current"), + "cancelling": enrichment._enrich_cancel.is_set(), } @@ -3250,7 +2321,7 @@ def api_enrichment_kick(): failures) — already-matched songs are left alone, so on a fully-matched library this is a fast no-op. Single-flight + coalescing like every other kick — spamming it queues at most one follow-up pass.""" - return {"started": _kick_enrich()} + return {"started": enrichment._kick_enrich()} @app.post("/api/enrichment/cancel") @@ -3259,9 +2330,9 @@ def api_enrichment_cancel(): halt after the current song (an in-flight ≤1/s lookup can't be interrupted, but no new one is started) and drop any coalesced follow-up. A no-op when nothing is running.""" - was_running = _enrich_status["running"] + was_running = enrichment._enrich_status["running"] if was_running: - _enrich_cancel.set() + enrichment._enrich_cancel.set() return {"ok": True, "was_running": was_running} @@ -3289,7 +2360,7 @@ def api_enrichment_rematch(data: dict = Body(...)): if meta_db.apply_enrichment_match(fn, h, "unscanned", allow_manual_overwrite=False): queued.append(fn) - started = _kick_enrich() if queued else False + started = enrichment._kick_enrich() if queued else False return {"queued": queued, "count": len(queued), "started": started} @@ -3305,8 +2376,8 @@ def api_enrichment_states(data: dict = Body(...)): fns = [str(f) for f in raw if isinstance(f, str)][:500] return { "states": meta_db.enrichment_states_for(fns), - "current": _enrich_status.get("current"), - "running": _enrich_status["running"], + "current": enrichment._enrich_status.get("current"), + "running": enrichment._enrich_status["running"], } @@ -3324,7 +2395,7 @@ def api_enrichment_refresh(filename: str): song["artist"], song["title"], song["album"], song["duration"]) meta_db.apply_enrichment_match(filename, h, "unscanned", allow_manual_overwrite=True) - return {"ok": True, "started": _kick_enrich()} + return {"ok": True, "started": enrichment._kick_enrich()} @app.get("/api/enrichment/review") @@ -3419,8 +2490,8 @@ def api_enrichment_search(artist: str = "", title: str = "", limit: int = 8, raise HTTPException(status_code=400, detail="artist or title required") limit = max(1, min(int(limit), 25)) try: - cands = _mb_search_recordings(artist, title, limit=limit) - except EnrichTransportError as e: + cands = enrichment._mb_search_recordings(artist, title, limit=limit) + except enrichment.EnrichTransportError as e: return JSONResponse({"error": "musicbrainz unavailable", "detail": str(e)}, status_code=503) ref = None @@ -3436,8 +2507,8 @@ def api_enrichment_search(artist: str = "", title: str = "", limit: int = 8, # romanized alias against the typed query ("Junko Ohashi") instead of # sinking to the bottom with a 0 artist score. try: - _alias_enrich(ref, cands) - except EnrichTransportError: + enrichment._alias_enrich(ref, cands) + except enrichment.EnrichTransportError: pass # aliases are a ranking nicety here; fall back to primary-name scoring return {"candidates": mb_match.rank_candidates(ref, cands)} @@ -3453,7 +2524,7 @@ async def api_enrichment_identify(request: Request): but the fpcalc Chromaprint binary is missing or the network is off. Async so the multipart is size-capped BEFORE spooling; the blocking fpcalc subprocess + AcoustID HTTP run in the threadpool via run_in_executor.""" - gate = _acoustid_gate() + gate = enrichment._acoustid_gate() if gate is not None: return gate # Pre-parse Content-Length guard — reject an oversized body before Starlette @@ -3465,10 +2536,10 @@ async def api_enrichment_identify(request: Request): cl_int = int(cl) except ValueError: return JSONResponse({"error": "Invalid Content-Length header"}, status_code=400) - if cl_int > _ACOUSTID_MAX_UPLOAD_BYTES + _MULTIPART_OVERHEAD_SLACK: + if cl_int > enrichment._ACOUSTID_MAX_UPLOAD_BYTES + _MULTIPART_OVERHEAD_SLACK: return JSONResponse({"error": "audio upload too large (256 MB max)"}, status_code=413) try: - form = await request.form(max_part_size=_ACOUSTID_MAX_UPLOAD_BYTES) + form = await request.form(max_part_size=enrichment._ACOUSTID_MAX_UPLOAD_BYTES) except Exception: return JSONResponse({"error": "audio upload too large (256 MB max)"}, status_code=413) file = form.get("file") @@ -3486,7 +2557,7 @@ async def api_enrichment_identify(request: Request): if not chunk: break total += len(chunk) - if total > _ACOUSTID_MAX_UPLOAD_BYTES: + if total > enrichment._ACOUSTID_MAX_UPLOAD_BYTES: return JSONResponse( {"error": "audio upload too large (256 MB max)"}, status_code=413) fh.write(chunk) @@ -3494,8 +2565,8 @@ async def api_enrichment_identify(request: Request): raise HTTPException(status_code=400, detail="empty upload") # fpcalc subprocess + AcoustID HTTP are blocking — off the event loop. cands = await asyncio.get_event_loop().run_in_executor( - None, _identify_by_fingerprint, tmp) - except EnrichTransportError as e: + None, enrichment._identify_by_fingerprint, tmp) + except enrichment.EnrichTransportError as e: return JSONResponse({"error": "acoustid unavailable", "detail": str(e)}, status_code=503) finally: @@ -3511,10 +2582,10 @@ def api_enrichment_identify_song(filename: str): the Fix-metadata / match-review flow). Same candidate shape as /search, so the review UI renders fingerprint hits like text hits. Same 412/503 gating; 404 when the song has no full-mix audio to fingerprint.""" - gate = _acoustid_gate() + gate = enrichment._acoustid_gate() if gate is not None: return gate - audio = _song_audio_file(filename) + audio = enrichment._song_audio_file(filename) if not audio: return JSONResponse( {"error": "no audio", @@ -3522,8 +2593,8 @@ def api_enrichment_identify_song(filename: str): "(a stems-only pack has no full mix to identify)."}, status_code=404) try: - cands = _identify_by_fingerprint(audio) - except EnrichTransportError as e: + cands = enrichment._identify_by_fingerprint(audio) + except enrichment.EnrichTransportError as e: return JSONResponse({"error": "acoustid unavailable", "detail": str(e)}, status_code=503) return {"candidates": cands} @@ -4570,18 +3641,18 @@ def _artist_links_payload(name: str, force: bool = False) -> dict: # The id is interpolated into the MB request path — same strict-shape rule # as the manifest identity keys (_MBID_RE), so a junk/hostile value stored # via a hand-rolled /pick body can never reach the request line. - if not mbid or not _MBID_RE.match(mbid): + if not mbid or not enrichment._MBID_RE.match(mbid): return {"links": {}, "matched": False} if not force: cached = meta_db.get_artist_enrichment(mbid) if cached: return {"links": cached["url_rels"], "genres": cached["genres"], "matched": True, "cached": True, "mb_artist_id": mbid} - if not _enrich_network_enabled(): + if not enrichment._enrich_network_enabled(): return {"links": {}, "matched": True, "offline": True, "mb_artist_id": mbid} try: - body = _mb_http_get(f"artist/{mbid}", {"inc": "url-rels+genres+tags"}) - except EnrichTransportError: + body = enrichment._mb_http_get(f"artist/{mbid}", {"inc": "url-rels+genres+tags"}) + except enrichment.EnrichTransportError: return {"links": {}, "matched": True, "offline": True, "mb_artist_id": mbid} links, genres = _artist_links_from_mb(body or {}) meta_db.put_artist_enrichment(mbid, links, genres) @@ -6137,8 +5208,8 @@ def api_art_cover_search(filename: str, q: str = ""): if not query: return {"query": "", "covers": []} try: - return {"query": query, "covers": _mb_search_release_groups(query, limit=8)} - except EnrichTransportError: + return {"query": query, "covers": enrichment._mb_search_release_groups(query, limit=8)} + except enrichment.EnrichTransportError: return {"query": query, "covers": [], "error": "unavailable"} @@ -6208,8 +5279,8 @@ def get_song_art_candidates(filename: str): if len(caa_entries) >= _ART_PICKER_MAX_CAA: break try: - imgs = _caa_index_cached(rid) - except EnrichTransportError: + imgs = enrichment._caa_index_cached(rid) + except enrichment.EnrichTransportError: # Offline / archive down — stop asking (each further miss would # only burn a timeout). The instant tiles still serve; a later # picker-open retries naturally (failures are never cached). @@ -6387,10 +5458,10 @@ def _gap_fill_proposals(cache_key: str, resolved) -> tuple[dict, str]: out["genres"] = genres # Identity keys (feedpak spec 1.14.0) — written in canonical form only. mbid = (row.get("mb_recording_id") or "").strip().lower() - if _MBID_RE.match(mbid) and _gap_fill_manifest_absent(manifest, "mbid"): + if enrichment._MBID_RE.match(mbid) and _gap_fill_manifest_absent(manifest, "mbid"): out["mbid"] = mbid isrc = (row.get("isrc") or "").strip().upper().replace("-", "").replace(" ", "") - if _ISRC_RE.match(isrc) and _gap_fill_manifest_absent(manifest, "isrc"): + if enrichment._ISRC_RE.match(isrc) and _gap_fill_manifest_absent(manifest, "isrc"): out["isrc"] = isrc return out, ("" if out else "nothing-missing") @@ -6606,8 +5677,8 @@ def _fetch_art_url(url: str) -> bytes: CAA) pins either — a bespoke pinned+SNI adapter here would be inconsistent and disproportionate. The cheap guards above still stop the realistic vectors (direct internal URL, redirect-to-internal).""" - if not _enrich_network_enabled(): - raise EnrichTransportError("art fetch disabled (offline)") + if not enrichment._enrich_network_enabled(): + raise enrichment.EnrichTransportError("art fetch disabled (offline)") import requests from urllib.parse import urljoin, urlparse for _hop in range(_ART_URL_MAX_REDIRECTS + 1): @@ -6620,16 +5691,16 @@ def _fetch_art_url(url: str) -> bytes: raise ValueError("url host is not allowed") try: with requests.get(url, timeout=15, stream=True, allow_redirects=False, - headers={"User-Agent": _enrich_user_agent()}) as resp: + headers={"User-Agent": enrichment._enrich_user_agent()}) as resp: if resp.status_code in (301, 302, 303, 307, 308): loc = resp.headers.get("Location") or "" if not loc: - raise EnrichTransportError( + raise enrichment.EnrichTransportError( f"HTTP {resp.status_code} without a Location") url = urljoin(url, loc) continue if resp.status_code != 200: - raise EnrichTransportError(f"HTTP {resp.status_code}") + raise enrichment.EnrichTransportError(f"HTTP {resp.status_code}") data = b"" for chunk in resp.iter_content(65536): data += chunk @@ -6637,8 +5708,8 @@ def _fetch_art_url(url: str) -> bytes: raise ValueError("image larger than 10 MB") return data except requests.RequestException as e: - raise EnrichTransportError(str(e)) from e - raise EnrichTransportError("too many redirects") + raise enrichment.EnrichTransportError(str(e)) from e + raise enrichment.EnrichTransportError("too many redirects") @app.post("/api/song/{filename:path}/art/url") @@ -6657,7 +5728,7 @@ def set_song_art_from_url(filename: str, data: dict): raise HTTPException(status_code=404, detail="unknown song") try: img_data = _fetch_art_url(url) - except EnrichTransportError as e: + except enrichment.EnrichTransportError as e: return JSONResponse({"error": "could not fetch image", "detail": str(e)}, status_code=502) except ValueError as e: diff --git a/tests/conftest.py b/tests/conftest.py index 5294711..ab364af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,36 @@ import structlog _LOGGING_NAMES = ("feedBack", "uvicorn", "uvicorn.error", "uvicorn.access") +@pytest.fixture(autouse=True) +def _reset_enrichment_state(): + """Reset the enrichment worker's process-global state between tests. + + The `server` fixtures pop-and-reimport `server`, but `lib/enrichment.py` + (which now owns the worker) stays imported for the whole session, so its + module globals — the cancel Event, the status dict, the caches — would + otherwise leak across tests. A test that set `_enrich_cancel` (or a stale + `running` status) could silently short-circuit a later direct + `_background_enrich()` call. Clear it up front so each test starts clean. + """ + try: + import enrichment + except ImportError: + yield + return + enrichment._enrich_cancel.clear() + enrichment._enrich_pending_pass = False + enrichment._enrich_status.update( + {"running": False, "processed": 0, "last_pass_at": None, + "total": 0, "matched": 0, "current": None}) + enrichment._enrich_last_fetch = 0.0 + enrichment._artist_alias_cache.clear() + # _caa_index_locks is deliberately left alone: it's guarded by + # _caa_index_locks_guard, so clearing it here (unlocked) would race a + # still-alive worker thread, and its entries are stateless per-release + # mutexes that don't leak test state anyway. + yield + + @pytest.fixture() def isolate_logging(): """Restore feedBack / uvicorn logger state after each test. diff --git a/tests/test_art_candidates.py b/tests/test_art_candidates.py index 5d6ba46..7f80ccf 100644 --- a/tests/test_art_candidates.py +++ b/tests/test_art_candidates.py @@ -12,6 +12,7 @@ tests/test_art_layer.py. """ import importlib +import enrichment import io as _io import sys @@ -119,8 +120,8 @@ def caa_index(server, monkeypatch): calls.append(release_id) return indexes.get(release_id) # unknown release → None (a CAA 404) fake.calls, fake.indexes = calls, indexes - monkeypatch.setattr(server, "_caa_release_index", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_caa_release_index", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) return fake @@ -274,10 +275,10 @@ def test_malicious_release_id_rejected_no_fetch_no_write(server, caa_index): """A crafted release id (path traversal) never matches _CAA_ID_RE, so it yields no images, opens no socket, and writes no cache file — inside the art dir or anywhere else.""" - art_dir = server._enrichment_art_dir() + art_dir = enrichment._enrichment_art_dir() before = set(art_dir.glob("*")) - assert not server._CAA_ID_RE.match("../../etc/x") - assert server._caa_index_cached("../../etc/x") == [] + assert not enrichment._CAA_ID_RE.match("../../etc/x") + assert enrichment._caa_index_cached("../../etc/x") == [] assert caa_index.calls == [] # the seam was never asked assert set(art_dir.glob("*")) == before # nothing written # And nothing landed at the traversal target beside the cache dir either. @@ -340,7 +341,7 @@ def test_fetch_art_url_follows_redirects_validating_each_hop(server, monkeypatch return _FakeResp(200, chunks=[b"IMGDATA"]) monkeypatch.setattr(requests, "get", fake_get) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) monkeypatch.setattr(server, "_url_host_is_internal", lambda u: (checked.append(u), False)[1]) data = server._fetch_art_url("https://coverartarchive.example/release/x/front-500") @@ -354,7 +355,7 @@ def test_fetch_art_url_blocks_redirect_to_internal(server, monkeypatch): import requests monkeypatch.setattr(requests, "get", lambda url, **kw: _FakeResp( 302, {"Location": "http://internal.example/x.png"})) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) monkeypatch.setattr(server, "_url_host_is_internal", lambda u: "internal" in u) with pytest.raises(ValueError): @@ -365,7 +366,7 @@ def test_fetch_art_url_redirect_budget(server, monkeypatch): import requests monkeypatch.setattr(requests, "get", lambda url, **kw: _FakeResp( 307, {"Location": "https://public.example/next.png"})) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) monkeypatch.setattr(server, "_url_host_is_internal", lambda u: False) - with pytest.raises(server.EnrichTransportError): + with pytest.raises(enrichment.EnrichTransportError): server._fetch_art_url("https://public.example/x.png") diff --git a/tests/test_art_layer.py b/tests/test_art_layer.py index fb641a2..0bb322b 100644 --- a/tests/test_art_layer.py +++ b/tests/test_art_layer.py @@ -7,6 +7,7 @@ here opens a socket, and the offline default is itself asserted. """ import importlib +import enrichment import io as _io import sys @@ -176,15 +177,15 @@ def caa(server, monkeypatch): calls.append(release_id) return art.get(release_id) fake.calls, fake.art = calls, art - monkeypatch.setattr(server, "_caa_http_get", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_caa_http_get", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) return fake def test_caa_fetch_fills_missing_art(server, client, caa): make_sloppak(server, "a.sloppak") # no pack art _match_row(server, "a.sloppak") - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["art_state"] == "caa" assert row["art_cache_path"] and row["art_cache_path"].endswith("caa_rel-1.jpg") @@ -193,7 +194,7 @@ def test_caa_fetch_fills_missing_art(server, client, caa): assert r.headers["content-type"] == "image/jpeg" # Settled: the next pass never re-fetches. n = len(caa.calls) - server._background_enrich() + enrichment._background_enrich() assert len(caa.calls) == n @@ -204,7 +205,7 @@ def test_caa_skips_pack_art_and_dedupes_by_release(server, caa): _match_row(server, "haspack.sloppak") _match_row(server, "b.sloppak") # same release as c _match_row(server, "c.sloppak") - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("haspack.sloppak")["art_state"] == "pack" assert server.meta_db.get_enrichment("b.sloppak")["art_state"] == "caa" assert server.meta_db.get_enrichment("c.sloppak")["art_state"] == "caa" @@ -214,10 +215,10 @@ def test_caa_skips_pack_art_and_dedupes_by_release(server, caa): def test_caa_404_marks_none(server, caa): make_sloppak(server, "a.sloppak") _match_row(server, "a.sloppak", release_id="rel-missing") - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["art_state"] == "none" n = len(caa.calls) - server._background_enrich() + enrichment._background_enrich() assert len(caa.calls) == n # never re-hammered @@ -226,13 +227,13 @@ def test_caa_transport_error_leaves_row_unevaluated(server, caa, monkeypatch): _match_row(server, "a.sloppak") def _down(release_id): - raise server.EnrichTransportError("down") - monkeypatch.setattr(server, "_caa_http_get", _down) - server._background_enrich() + raise enrichment.EnrichTransportError("down") + monkeypatch.setattr(enrichment, "_caa_http_get", _down) + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["art_state"] is None # Network back → next pass completes it. - monkeypatch.setattr(server, "_caa_http_get", caa) - server._background_enrich() + monkeypatch.setattr(enrichment, "_caa_http_get", caa) + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["art_state"] == "caa" @@ -240,22 +241,22 @@ def test_offline_default_skips_art_worker(server, monkeypatch): """Under the plain test env the whole art phase is skipped with the rest of the network work.""" calls = [] - monkeypatch.setattr(server, "_caa_http_get", lambda rid: calls.append(rid)) + monkeypatch.setattr(enrichment, "_caa_http_get", lambda rid: calls.append(rid)) make_sloppak(server, "a.sloppak") _match_row(server, "a.sloppak") - server._background_enrich() + enrichment._background_enrich() assert calls == [] assert server.meta_db.get_enrichment("a.sloppak")["art_state"] is None def test_lru_prune_evicts_oldest_and_resets_rows(server, caa, monkeypatch): - monkeypatch.setattr(server, "_CAA_CACHE_CAP_BYTES", 1) # everything over cap + monkeypatch.setattr(enrichment, "_CAA_CACHE_CAP_BYTES", 1) # everything over cap make_sloppak(server, "a.sloppak", title="One") make_sloppak(server, "b.sloppak", title="Two") caa.art["rel-2"] = png_bytes((1, 1, 1)) _match_row(server, "a.sloppak", release_id="rel-1") _match_row(server, "b.sloppak", release_id="rel-2") - server._background_enrich() + enrichment._background_enrich() # With a 1-byte cap every fetch immediately evicts — the rows that pointed # at evicted files were reset to unevaluated. caa_files = list(server.ART_CACHE_DIR.glob("caa_*.jpg")) @@ -282,13 +283,13 @@ def test_delete_override_restores_caa_fallback(server, client, caa): _match_row(server, "a.sloppak") # Pin an override BEFORE the art worker runs → the pass stamps art_state='user'. client.post("/api/song/a.sloppak/art/upload", json={"image": b64(png_bytes())}) - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["art_state"] == "user" # Remove it → the row resets to unevaluated… assert client.delete("/api/art/a.sloppak/override").json()["removed"] assert server.meta_db.get_enrichment("a.sloppak")["art_state"] is None # …and the next pass fetches + serves the release's front cover. - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["art_state"] == "caa" r = client.get("/api/song/a.sloppak/art") assert r.status_code == 200 diff --git a/tests/test_artist_page.py b/tests/test_artist_page.py index 2b53237..7397f0d 100644 --- a/tests/test_artist_page.py +++ b/tests/test_artist_page.py @@ -10,7 +10,7 @@ Two halves, mirroring the design's split: * GET /api/artist/{name}/links + POST .../links/refresh — the lazy, cached, opt-in external-links layer. The HTTP transport is a fake over - `server._mb_http_get` (the ONE network seam — same pattern as + `enrichment._mb_http_get` (the ONE network seam — same pattern as tests/test_mb_enrichment.py), so nothing here opens a socket. Covers the url-rel whitelist mapping, the http(s) scheme gate (a hostile javascript: resource never reaches a link slot), cache-hit second calls making no @@ -19,6 +19,7 @@ Two halves, mirroring the design's split: """ import importlib +import enrichment import json import sys from urllib.parse import quote @@ -88,7 +89,7 @@ class FakeMBArtist: def __call__(self, path, params): if self.raise_transport: - raise self._srv.EnrichTransportError("fake network down") + raise enrichment.EnrichTransportError("fake network down") self.calls.append((path, dict(params))) if path == f"artist/{MBID}": return self.doc @@ -100,8 +101,8 @@ def mb_artist(server, monkeypatch): """Install the fake transport AND enable the network flag (the test env disables it by default — see test_links_offline_returns_empty).""" fake = FakeMBArtist(server) - monkeypatch.setattr(server, "_mb_http_get", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_mb_http_get", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) return fake diff --git a/tests/test_context_menu_api.py b/tests/test_context_menu_api.py index fdc1792..207bea5 100644 --- a/tests/test_context_menu_api.py +++ b/tests/test_context_menu_api.py @@ -4,6 +4,7 @@ contents). The refresh flow reuses the P8 fake-transport pattern — nothing here opens a socket.""" import importlib +import enrichment import sys import pytest @@ -85,9 +86,9 @@ def test_refresh_resets_even_a_manual_pin_and_rematches(server, client, monkeypa "status": "Official", "date": "1990-09-24", "release-group": {"primary-type": "Album"}}], }]} - monkeypatch.setattr(server, "_mb_http_get", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) - server._background_enrich() + monkeypatch.setattr(enrichment, "_mb_http_get", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["mb_recording_id"] == "rec-new" diff --git a/tests/test_enrichment_plumbing.py b/tests/test_enrichment_plumbing.py index eb88f77..5f670cb 100644 --- a/tests/test_enrichment_plumbing.py +++ b/tests/test_enrichment_plumbing.py @@ -5,6 +5,7 @@ contracts it will inherit: rename-survivable idempotent hashing, manual rows never auto-reset, never purged on rescan, purged on explicit delete.""" import importlib +import enrichment import sys import pytest @@ -58,7 +59,7 @@ def test_pending_covers_new_unscanned_and_changed(server): _put(server, "a.archive") assert [r["filename"] for r in server.meta_db.enrichment_pending()] == ["a.archive"] # stubbed → still unscanned → still pending (the matcher hasn't run) - server._background_enrich() + enrichment._background_enrich() assert [r["filename"] for r in server.meta_db.enrichment_pending()] == ["a.archive"] # a matched row with the CURRENT hash is settled… h = server.meta_db.enrichment_content_hash("Artist", "Song", "", 100) @@ -76,7 +77,7 @@ def test_pending_covers_new_unscanned_and_changed(server): def test_hash_change_resets_matched_but_never_manual(server): _put(server, "a.archive") _put(server, "b.archive", title="Other") - server._background_enrich() + enrichment._background_enrich() with server.meta_db._lock: server.meta_db.conn.execute( "UPDATE song_enrichment SET match_state = 'matched' WHERE filename = 'a.archive'") @@ -86,7 +87,7 @@ def test_hash_change_resets_matched_but_never_manual(server): # identity edits… _put(server, "a.archive", title="Song v2") _put(server, "b.archive", title="Other v2") - server._background_enrich() + enrichment._background_enrich() a = server.meta_db.get_enrichment("a.archive") b = server.meta_db.get_enrichment("b.archive") # …drop a stale MATCH back to unscanned with the fresh hash @@ -99,7 +100,7 @@ def test_hash_change_resets_matched_but_never_manual(server): def test_failed_rows_not_requeued_by_pending(server): _put(server, "a.archive") - server._background_enrich() + enrichment._background_enrich() with server.meta_db._lock: server.meta_db.conn.execute( "UPDATE song_enrichment SET match_state = 'failed' WHERE filename = 'a.archive'") @@ -113,7 +114,7 @@ def test_failed_rows_not_requeued_by_pending(server): def test_enrich_pass_stamps_every_song(server): for i in range(5): _put(server, f"s{i}.archive", title=f"Song {i}") - server._background_enrich() + enrichment._background_enrich() for i in range(5): row = server.meta_db.get_enrichment(f"s{i}.archive") assert row is not None @@ -126,7 +127,7 @@ def test_enrich_pass_stamps_every_song(server): def test_rescan_never_purges_enrichment(server): _put(server, "a.archive") - server._background_enrich() + enrichment._background_enrich() server.meta_db.delete_missing(set()) # file vanished from a scan snapshot assert server.meta_db.get_enrichment("a.archive") is not None # row survives # …and is invisible in the read-time-filtered counts @@ -138,7 +139,7 @@ def test_rescan_never_purges_enrichment(server): def test_status_endpoint_counts(client, server): _put(server, "a.archive") _put(server, "b.archive", title="Other") - server._background_enrich() + enrichment._background_enrich() body = client.get("/api/enrichment/status").json() assert body["states"] == {"unscanned": 2} assert body["total_songs"] == 2 @@ -147,7 +148,7 @@ def test_status_endpoint_counts(client, server): def test_art_cache_dir_created(server): - d = server._enrichment_art_dir() + d = enrichment._enrichment_art_dir() assert d.is_dir() assert d.name == "art_cache" @@ -156,7 +157,7 @@ def test_art_cache_dir_created(server): def test_states_for_returns_only_known_filenames(server): _put(server, "a.archive") - server._background_enrich() + enrichment._background_enrich() got = server.meta_db.enrichment_states_for(["a.archive", "nope.archive"]) assert got == {"a.archive": "unscanned"} # unknown filename absent assert server.meta_db.enrichment_states_for([]) == {} @@ -165,7 +166,7 @@ def test_states_for_returns_only_known_filenames(server): def test_states_endpoint(client, server): _put(server, "a.archive") _put(server, "b.archive", title="Other") - server._background_enrich() + enrichment._background_enrich() body = client.post("/api/enrichment/states", json={"filenames": ["a.archive", "zzz.missing"]}).json() assert body["states"] == {"a.archive": "unscanned"} @@ -175,7 +176,7 @@ def test_states_endpoint(client, server): def test_status_exposes_progress_fields(client, server): _put(server, "a.archive") - server._background_enrich() + enrichment._background_enrich() body = client.get("/api/enrichment/status").json() for k in ("total", "matched", "current", "cancelling"): assert k in body @@ -186,7 +187,7 @@ def test_cancel_is_noop_when_idle(client, server): body = client.post("/api/enrichment/cancel").json() assert body == {"ok": True, "was_running": False} # A no-op must not arm the flag (which would then poison the next pass). - assert server._enrich_cancel.is_set() is False + assert enrichment._enrich_cancel.is_set() is False def test_cancel_flag_halts_matching_loop_between_songs(server, monkeypatch): @@ -195,28 +196,28 @@ def test_cancel_flag_halts_matching_loop_between_songs(server, monkeypatch): # Force the matcher path on (the test env is offline by default) and stub the # per-song matcher so nothing touches the network — it just trips Stop after # the first song, exactly as the /cancel route would mid-pass. - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) calls = [] def fake_enrich_one(row, **_kw): calls.append(row["filename"]) - server._enrich_cancel.set() + enrichment._enrich_cancel.set() - monkeypatch.setattr(server, "_enrich_one", fake_enrich_one) - server._enrich_cancel.clear() - server._background_enrich() + monkeypatch.setattr(enrichment, "_enrich_one", fake_enrich_one) + enrichment._enrich_cancel.clear() + enrichment._background_enrich() # The loop checks cancel BEFORE each song, so exactly one is processed before # it breaks — not the whole 4-row queue. assert calls == ["s0.archive"] - assert server._enrich_status["total"] == 4 - assert server._enrich_status["matched"] == 1 + assert enrichment._enrich_status["total"] == 4 + assert enrichment._enrich_status["matched"] == 1 def test_rematch_requeues_visible_but_skips_manual(server, client): _put(server, "a.archive") # will be 'matched' _put(server, "b.archive", title="Other") # will be 'failed' _put(server, "c.archive", title="Pinned") # will be 'manual' — untouchable - server._background_enrich() + enrichment._background_enrich() with server.meta_db._lock: server.meta_db.conn.execute( "UPDATE song_enrichment SET match_state='matched' WHERE filename='a.archive'") @@ -240,7 +241,7 @@ def test_rematch_requeues_visible_but_skips_manual(server, client): # ── filename-derived artist/title fallback (blank-artist packs) ─────────────── def test_filename_artist_title_parse(server): - f = server._artist_title_from_filename + f = enrichment._artist_title_from_filename assert f("CDLC/0 - City Pop/Tatsuro-Yamashita_Ride-On-Time_v1_p.feedpak") == \ {"artist": "Tatsuro Yamashita", "title": "Ride On Time"} assert f("Anri_Windy-Summer_v1_p.feedpak") == {"artist": "Anri", "title": "Windy Summer"} @@ -255,18 +256,18 @@ def test_blank_artist_seeds_match_from_filename(server, monkeypatch): server.meta_db.put("Tatsuro-Yamashita_Ride-On-Time_v1_p.feedpak", 0, 0, { "title": "Tatsuro-Yamashita_Ride-On-Time_v1_p", "artist": "", "album": "", "duration": 240, "arrangements": [{"name": "Bass", "index": 0}]}) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) - monkeypatch.setattr(server, "_manifest_exact_ids", lambda fn: {}) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_manifest_exact_ids", lambda fn: {}) seen = {} def fake_search(artist, title, limit=8): seen["artist"], seen["title"] = artist, title return [] - monkeypatch.setattr(server, "_mb_search_recordings", fake_search) + monkeypatch.setattr(enrichment, "_mb_search_recordings", fake_search) row = next(r for r in server.meta_db.enrichment_pending() if r["filename"].startswith("Tatsuro")) - server._enrich_one(row) + enrichment._enrich_one(row) # the blank pack artist was replaced by the filename-derived identity for # the search (this is exactly what rescues the 'failed' pile) assert seen == {"artist": "Tatsuro Yamashita", "title": "Ride On Time"} @@ -276,18 +277,18 @@ def test_present_artist_is_not_overridden_by_filename(server, monkeypatch): server.meta_db.put("Weird-Filename_x_y.feedpak", 0, 0, { "title": "Real Title", "artist": "Real Artist", "album": "", "duration": 100, "arrangements": [{"name": "Lead", "index": 0}]}) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) - monkeypatch.setattr(server, "_manifest_exact_ids", lambda fn: {}) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_manifest_exact_ids", lambda fn: {}) seen = {} def fake_search(artist, title, limit=8): seen["artist"], seen["title"] = artist, title return [] - monkeypatch.setattr(server, "_mb_search_recordings", fake_search) + monkeypatch.setattr(enrichment, "_mb_search_recordings", fake_search) row = next(r for r in server.meta_db.enrichment_pending() if r["filename"].startswith("Weird")) - server._enrich_one(row) + enrichment._enrich_one(row) # a pack that DOES carry an artist keeps it — the filename is never consulted assert seen == {"artist": "Real Artist", "title": "Real Title"} @@ -295,7 +296,7 @@ def test_present_artist_is_not_overridden_by_filename(server, monkeypatch): def test_kick_clears_a_stale_cancel(server): # A cancelled-then-rekicked pass must start clean: _kick_enrich clears the # flag so the fresh pass isn't aborted the instant it checks. - server._enrich_cancel.set() - server._kick_enrich() + enrichment._enrich_cancel.set() + enrichment._kick_enrich() server._join_background_db_threads() - assert server._enrich_cancel.is_set() is False + assert enrichment._enrich_cancel.is_set() is False diff --git a/tests/test_field_overrides.py b/tests/test_field_overrides.py index 8761aab..8f46b12 100644 --- a/tests/test_field_overrides.py +++ b/tests/test_field_overrides.py @@ -6,6 +6,7 @@ song (delete_song). Locks pin a field against a later auto-match. """ import importlib +import enrichment import sys import pytest @@ -149,7 +150,7 @@ def test_locked_fields_reader(server): def test_compose_lock_filter_strips_locked_cand_keys(server): - f = server._compose_lock_filter(None, {"artist", "year"}) + f = enrichment._compose_lock_filter(None, {"artist", "year"}) cand = {"recording_id": "r", "artist": "X", "artist_sort": "X", "title": "T", "year": "1990", "album": "A", "genres": ["rock"]} out = f(cand) @@ -158,7 +159,7 @@ def test_compose_lock_filter_strips_locked_cand_keys(server): # …identity + unlocked display fields survive assert out["recording_id"] == "r" and out["title"] == "T" and out["album"] == "A" # no locks → base filter returned unchanged (zero-copy common path) - assert server._compose_lock_filter(None, set()) is None + assert enrichment._compose_lock_filter(None, set()) is None # ── display overlay in the grid (slice 3) ───────────────────────────────────── diff --git a/tests/test_mb_enrichment.py b/tests/test_mb_enrichment.py index d0937ca..a3c78d5 100644 --- a/tests/test_mb_enrichment.py +++ b/tests/test_mb_enrichment.py @@ -1,6 +1,6 @@ """Server-level tests for the P8 MusicBrainz matcher + Match-Review flow. -The HTTP transport is a fake installed over `server._mb_http_get` — the ONE +The HTTP transport is a fake installed over `enrichment._mb_http_get` — the ONE seam enrichment uses to reach the network — so nothing here ever opens a socket. The offline default is itself under test: without explicitly enabling the network flag, a pass must skip matching entirely (pytest can @@ -8,6 +8,7 @@ never hit MusicBrainz, whatever a test triggers). """ import importlib +import enrichment import sys import pytest @@ -50,7 +51,7 @@ class FakeMB: def __call__(self, path, params): if self.raise_transport: - raise self._srv.EnrichTransportError("fake network down") + raise enrichment.EnrichTransportError("fake network down") self.calls.append((path, dict(params))) if path == "recording": return self.search_response @@ -71,8 +72,8 @@ def mb(server, monkeypatch): disables it by default — see test_offline_default_skips_matching).""" fake = FakeMB() fake._srv = server - monkeypatch.setattr(server, "_mb_http_get", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_mb_http_get", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) return fake @@ -111,8 +112,8 @@ def test_search_falls_back_to_loose_when_strict_is_empty(server, monkeypatch): return {"recordings": []} return {"recordings": [mb_doc(rid="rec-x", title="Telephone Number")]} - monkeypatch.setattr(server, "_mb_http_get", _routed) - cands = server._mb_search_recordings("Junko Ohashi", "Telephone Number") + monkeypatch.setattr(enrichment, "_mb_http_get", _routed) + cands = enrichment._mb_search_recordings("Junko Ohashi", "Telephone Number") assert len(cands) == 1 assert len(calls) == 2 # strict first, then the loose retry assert calls[0].startswith("recording:") # strict is the field-phrase form @@ -128,8 +129,8 @@ def test_search_does_not_retry_when_strict_hits(server, monkeypatch): calls.append(params.get("query", "")) return {"recordings": [mb_doc()]} - monkeypatch.setattr(server, "_mb_http_get", _routed) - cands = server._mb_search_recordings("AC/DC", "Thunderstruck") + monkeypatch.setattr(enrichment, "_mb_http_get", _routed) + cands = enrichment._mb_search_recordings("AC/DC", "Thunderstruck") assert len(cands) == 1 assert len(calls) == 1 @@ -147,10 +148,10 @@ def test_artist_aliases_fetched_and_cached(server, monkeypatch): return {"sort-name": "Ohashi, Junko", "aliases": [{"name": "Junko Ohashi"}, {"name": "大橋 純子"}]} - monkeypatch.setattr(server, "_mb_http_get", fake) - names = server._mb_artist_aliases(_AID) + monkeypatch.setattr(enrichment, "_mb_http_get", fake) + names = enrichment._mb_artist_aliases(_AID) assert "Junko Ohashi" in names and "Ohashi, Junko" in names - server._mb_artist_aliases(_AID) # cached → no second request + enrichment._mb_artist_aliases(_AID) # cached → no second request assert len(calls) == 1 @@ -158,8 +159,8 @@ def test_artist_aliases_rejects_bad_id(server, monkeypatch): def boom(path, params): raise AssertionError("must not fetch for a non-UUID id") - monkeypatch.setattr(server, "_mb_http_get", boom) - assert server._mb_artist_aliases("not-a-uuid") == [] + monkeypatch.setattr(enrichment, "_mb_http_get", boom) + assert enrichment._mb_artist_aliases("not-a-uuid") == [] def test_enrich_auto_matches_japanese_primary_via_alias(server, monkeypatch): @@ -176,9 +177,9 @@ def test_enrich_auto_matches_japanese_primary_via_alias(server, monkeypatch): return {"recordings": [mb_doc(rid="rec-jp", title="Telephone Number", artist="大橋純子", artist_id=_AID)]} # loose hit - monkeypatch.setattr(server, "_mb_http_get", _routed) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) - server._background_enrich() + monkeypatch.setattr(enrichment, "_mb_http_get", _routed) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) + enrichment._background_enrich() row = server.meta_db.get_enrichment("x.sloppak") # The romanized alias lifts the artist over the auto floor → auto-confirmed. assert row["match_state"] == "matched" @@ -190,10 +191,10 @@ def test_enrich_auto_matches_japanese_primary_via_alias(server, monkeypatch): def test_locked_field_not_canonicalized_by_auto_match(server, monkeypatch): _put(server, "x.sloppak") # title "Thunderstruck (v2)", artist "ACDC" server.meta_db.set_song_override("x.sloppak", "artist", locked=True) - monkeypatch.setattr(server, "_mb_http_get", + monkeypatch.setattr(enrichment, "_mb_http_get", lambda path, params: {"recordings": [mb_doc()]}) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) - server._background_enrich() + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) + enrichment._background_enrich() row = server.meta_db.get_enrichment("x.sloppak") assert row["match_state"] == "matched" # still matches (identity applies)… assert row["canon_artist"] is None # …but the LOCKED artist isn't canonicalized @@ -208,9 +209,9 @@ def test_offline_default_skips_matching(server, monkeypatch): but never matches — even with a transport installed.""" fake = FakeMB() fake._srv = server - monkeypatch.setattr(server, "_mb_http_get", fake) + monkeypatch.setattr(enrichment, "_mb_http_get", fake) _put(server, "a.sloppak") - server._background_enrich() + enrichment._background_enrich() assert fake.calls == [] assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "unscanned" @@ -218,15 +219,15 @@ def test_offline_default_skips_matching(server, monkeypatch): def test_real_transport_refuses_when_offline(server): """_mb_http_get itself raises (before any socket) when the network is disabled — defence in depth under pytest.""" - with pytest.raises(server.EnrichTransportError): - server._mb_http_get("recording", {"query": "x"}) + with pytest.raises(enrichment.EnrichTransportError): + enrichment._mb_http_get("recording", {"query": "x"}) def test_transport_error_pauses_pass_without_burning_attempts(server, mb): _put(server, "a.sloppak") _put(server, "b.sloppak", title="Other Song") mb.raise_transport = True - server._background_enrich() + enrichment._background_enrich() for fn in ("a.sloppak", "b.sloppak"): row = server.meta_db.get_enrichment(fn) assert row["match_state"] == "unscanned" @@ -234,7 +235,7 @@ def test_transport_error_pauses_pass_without_burning_attempts(server, mb): # Network comes back → the next kick matches both. mb.raise_transport = False mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "matched" @@ -243,7 +244,7 @@ def test_transport_error_pauses_pass_without_burning_attempts(server, mb): def test_high_confidence_auto_matches_and_settles(server, mb): _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["match_source"] == "text" @@ -256,13 +257,13 @@ def test_high_confidence_auto_matches_and_settles(server, mb): assert row["genres"] == ["hard rock"] # Settled: another pass makes NO further network calls… n = len(mb.calls) - server._background_enrich() + enrichment._background_enrich() assert len(mb.calls) == n # …until the identity changes, which re-matches. _put(server, "a.sloppak", title="Back in Black") mb.search_response = {"recordings": [mb_doc(rid="rec-2", title="Back in Black", album="Back in Black", date="1980-07-25")]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["mb_recording_id"] == "rec-2" @@ -271,7 +272,7 @@ def test_medium_confidence_goes_to_review_not_canonical(server, mb): # Partial artist agreement → medium confidence. _put(server, "a.sloppak", artist="AC/DC ft Nobody") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "review" assert row["match_source"] == "text" @@ -281,7 +282,7 @@ def test_medium_confidence_goes_to_review_not_canonical(server, mb): assert row["candidates"] and row["candidates"][0]["recording_id"] == "rec-1" # A review row is settled while its identity is unchanged — no re-query. n = len(mb.calls) - server._background_enrich() + enrichment._background_enrich() assert len(mb.calls) == n @@ -289,14 +290,14 @@ def test_low_confidence_fails_with_backoff(server, mb): _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc(rid="rec-x", title="Sunrise", artist="Norah Jones")]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "failed" assert row["attempts"] == 1 assert row["last_attempt_at"] is not None # Immediately after, the backoff hasn't elapsed → no retry, no network. n = len(mb.calls) - server._background_enrich() + enrichment._background_enrich() assert len(mb.calls) == n assert server.meta_db.get_enrichment("a.sloppak")["attempts"] == 1 # Rewind the clock two hours → eligible again, attempts increments. @@ -304,7 +305,7 @@ def test_low_confidence_fails_with_backoff(server, mb): server.meta_db.conn.execute( "UPDATE song_enrichment SET last_attempt_at = last_attempt_at - 7200") server.meta_db.conn.commit() - server._background_enrich() + enrichment._background_enrich() assert len(mb.calls) == n + 1 assert server.meta_db.get_enrichment("a.sloppak")["attempts"] == 2 @@ -312,7 +313,7 @@ def test_low_confidence_fails_with_backoff(server, mb): def test_no_results_fails(server, mb): _put(server, "a.sloppak") mb.search_response = {"recordings": []} - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "failed" @@ -322,7 +323,7 @@ def test_cache_hit_copies_match_without_network(server, mb): _put(server, "a.sloppak") _put(server, "b.sloppak") # identical identity → same content_hash mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() assert len(mb.search_calls) == 1 # ONE search covered both charts a = server.meta_db.get_enrichment("a.sloppak") b = server.meta_db.get_enrichment("b.sloppak") @@ -347,7 +348,7 @@ def test_manifest_mbid_tier0(server, mb): _write_sloppak_manifest(server, "a.sloppak", f"mbid: {mbid}\n") _put(server, "a.sloppak") mb.recording_lookups[mbid] = mb_doc(rid=mbid) - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["match_source"] == "mbid" @@ -360,7 +361,7 @@ def test_manifest_isrc_tier1(server, mb): _write_sloppak_manifest(server, "a.sloppak", "isrc: AUAP09000045\n") _put(server, "a.sloppak") mb.isrc_lookups["AUAP09000045"] = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["match_source"] == "isrc" @@ -374,7 +375,7 @@ def test_manifest_isrc_display_hyphens_stripped(server, mb): _write_sloppak_manifest(server, "a.sloppak", "isrc: AU-AP0-90-00045\n") _put(server, "a.sloppak") mb.isrc_lookups["AUAP09000045"] = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["match_source"] == "isrc" @@ -387,7 +388,7 @@ def test_bad_manifest_mbid_falls_through_to_text(server, mb): _put(server, "a.sloppak") mb.recording_lookups.clear() # lookup 404s (typo'd manifest) mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["match_source"] == "text" @@ -401,7 +402,7 @@ def test_manual_never_overwritten_by_matcher(server, mb): "a.sloppak", {"recording_id": "user-pick", "title": "Thunderstruck", "artist": "AC/DC"}, source="search") mb.search_response = {"recordings": [mb_doc(rid="machine-pick")]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "manual" assert row["mb_recording_id"] == "user-pick" @@ -419,7 +420,7 @@ def _seed_review(server, mb, fn="a.sloppak", title="Thunderstruck (v2)"): # legitimately copies an earlier row instead of running the text tiers). _put(server, fn, title=title, artist="AC/DC ft Nobody") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment(fn)["match_state"] == "review" @@ -457,11 +458,11 @@ def test_review_reject_route_never_retries(server, mb, client): assert row["match_source"] == "rejected" # Rejected rows are excluded from the retry backoff forever… n = len(mb.calls) - server._background_enrich() + enrichment._background_enrich() assert len(mb.calls) == n # …but an identity edit re-queues (the user fixed the metadata). _put(server, "a.sloppak", artist="AC/DC") - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "matched" # Rejecting a manual row is refused. r = client.post("/api/enrichment/review/a.sloppak/reject") @@ -502,8 +503,8 @@ def test_search_proxy(server, mb, client, monkeypatch): assert body["candidates"][0]["score"] > 0.9 # Transport failure surfaces as 503, not a 500. def _down(path, params): - raise server.EnrichTransportError("down") - monkeypatch.setattr(server, "_mb_http_get", _down) + raise enrichment.EnrichTransportError("down") + monkeypatch.setattr(enrichment, "_mb_http_get", _down) r = client.get("/api/enrichment/search", params={"title": "x"}) assert r.status_code == 503 @@ -523,8 +524,8 @@ def test_match_facet_filters_grid_and_stats(server, mb, client, monkeypatch): if "revsong" in q: return {"recordings": [mb_doc(rid="rec-r", title="Revsong")]} return {"recordings": []} - monkeypatch.setattr(server, "_mb_http_get", _routed) - server._background_enrich() + monkeypatch.setattr(enrichment, "_mb_http_get", _routed) + enrichment._background_enrich() # Pendsong got failed by the pass (no results); reset it to unscanned to # represent the not-yet-scanned band. with server.meta_db._lock: @@ -566,14 +567,14 @@ def test_auto_threshold_setting_moves_the_auto_review_boundary(server, mb, clien client.post("/api/settings", json={"enrich_auto_threshold": 0.95}) _put(server, "a.sloppak", title="Highway to Hell", artist="AC/DC", year="", duration=0) - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "review" # Lower the bar to the default 0.90 → an identity edit re-queues, and the # same 0.90-scored candidate now auto-applies. client.post("/api/settings", json={"enrich_auto_threshold": 0.9}) _put(server, "a.sloppak", title="Highway to Hell", artist="AC/DC", year="", duration=0, album="Different Album") - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert abs(row["match_score"] - 0.9) < 1e-6 @@ -583,7 +584,7 @@ def test_enrich_enabled_setting_gates_background_matching(server, mb, client): client.post("/api/settings", json={"enrich_enabled": False}) _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() assert mb.calls == [] assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "unscanned" # Manual search/fix stays available while the background matcher is off. @@ -591,7 +592,7 @@ def test_enrich_enabled_setting_gates_background_matching(server, mb, client): assert r.status_code == 200 # Re-enable → the next pass matches. client.post("/api/settings", json={"enrich_enabled": True}) - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "matched" @@ -625,7 +626,7 @@ def test_review_queue_orders_missing_data_first(server, mb, client): _seed_review(server, mb, fn="aa.sloppak", title="Thunderstruck (v2)") _put(server, "zz.sloppak", title="Thunderstruck (Live)", artist="AC/DC ft Nobody", album="", year="") - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("zz.sloppak")["match_state"] == "review" songs = client.get("/api/enrichment/review").json()["songs"] assert [s["filename"] for s in songs] == ["zz.sloppak", "aa.sloppak"] diff --git a/tests/test_scraper_options.py b/tests/test_scraper_options.py index 03859a7..fbcf28f 100644 --- a/tests/test_scraper_options.py +++ b/tests/test_scraper_options.py @@ -8,6 +8,7 @@ flag is only force-enabled where a test needs the pipeline to run. """ import importlib +import enrichment import io as _io import sys @@ -57,8 +58,8 @@ class FakeMB: @pytest.fixture() def mb(server, monkeypatch): fake = FakeMB() - monkeypatch.setattr(server, "_mb_http_get", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_mb_http_get", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) return fake @@ -116,13 +117,13 @@ def test_musicbrainz_source_off_stamps_without_matching(server, mb, client): client.post("/api/settings", json={"enrich_src_musicbrainz": False}) _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() assert mb.calls == [] row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "unscanned" # hash stamped, no match # Re-enabling picks the same row up on the next pass. client.post("/api/settings", json={"enrich_src_musicbrainz": True}) - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "matched" @@ -133,7 +134,7 @@ def test_field_toggles_strip_auto_applied_fields(server, mb, client): "enrich_apply_genres": False}) _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["canon_artist"] == "AC/DC" @@ -150,7 +151,7 @@ def test_names_toggle_keeps_ids_and_other_fields(server, mb, client): client.post("/api/settings", json={"enrich_apply_names": False}) _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["canon_artist"] is None @@ -170,7 +171,7 @@ def test_review_accept_applies_all_fields_despite_toggles(server, mb, client): # Partial artist agreement → review tier (candidates stored unfiltered). _put(server, "a.sloppak", artist="AC/DC ft Nobody") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["match_state"] == "review" r = client.post("/api/enrichment/review/a.sloppak/accept", json={"recording_id": "rec-1"}) @@ -192,21 +193,21 @@ def test_reenabling_field_backfills_matched_row(server, mb, client): client.post("/api/settings", json={"enrich_apply_year": False}) _put(server, "a.sloppak") mb.search_response = {"recordings": [mb_doc()]} - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["canon_year"] is None # suppressed assert row["apply_mask"] == "enrich_apply_year" # …and remembered # Re-enable → next pass re-queues and backfills the year (hash unchanged). client.post("/api/settings", json={"enrich_apply_year": True}) - server._background_enrich() + enrichment._background_enrich() row = server.meta_db.get_enrichment("a.sloppak") assert row["match_state"] == "matched" assert row["canon_year"] == "1990" # backfilled assert row["apply_mask"] in (None, "") # fully applied now # Converged: a fully-applied row is not re-queued again. assert server.meta_db.enrichment_pending( - allowed_keys=frozenset(server._ENRICH_APPLY_FIELDS)) == [] + allowed_keys=frozenset(enrichment._ENRICH_APPLY_FIELDS)) == [] def test_partial_match_is_not_a_cache_donor(server): @@ -271,8 +272,8 @@ def caa(server, monkeypatch): calls.append(release_id) return art.get(release_id) fake.calls = calls - monkeypatch.setattr(server, "_caa_http_get", fake) - monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True) + monkeypatch.setattr(enrichment, "_caa_http_get", fake) + monkeypatch.setattr(enrichment, "_enrich_network_enabled", lambda: True) return fake @@ -280,13 +281,13 @@ def test_caa_source_toggle_gates_art_fetch(server, client, caa): make_sloppak(server, "a.sloppak") _match_row(server, "a.sloppak") client.post("/api/settings", json={"enrich_src_caa": False}) - server._background_enrich() + enrichment._background_enrich() assert caa.calls == [] row = server.meta_db.get_enrichment("a.sloppak") assert row["art_state"] is None # not forfeited, just skipped # Re-enable → the same row is picked up. client.post("/api/settings", json={"enrich_src_caa": True}) - server._background_enrich() + enrichment._background_enrich() assert server.meta_db.get_enrichment("a.sloppak")["art_state"] == "caa" @@ -294,7 +295,7 @@ def test_apply_art_toggle_gates_art_fetch(server, client, caa): make_sloppak(server, "a.sloppak") _match_row(server, "a.sloppak") client.post("/api/settings", json={"enrich_apply_art": False}) - server._background_enrich() + enrichment._background_enrich() assert caa.calls == [] assert server.meta_db.get_enrichment("a.sloppak")["art_state"] is None