feedBack/lib/scan.py

"""The library scanner: the background scan, its process pool, and the kick/runner
plumbing that serialises passes.

Carved VERBATIM out of server.py (R3b) except the seam reads. Everything shared is read
LATE off appstate — the same contract every module in lib/routers/ uses, and it is not
cosmetic: tests monkeypatch CONFIG_DIR and swap meta_db, so a value captured at import
time would pin the wrong one for the life of the process.

    CONFIG_DIR        -> appstate.config_dir
    meta_db           -> appstate.meta_db
    _default_settings -> appstate.default_settings()
    _stat_for_cache   -> appstate.stat_for_cache()
    _feedBack_server_root() -> appstate.server_root      <- see below

━━━ THE SCAN STATUS IS REBOUND, NOT MUTATED ━━━

`_background_scan` does `global _scan_status; _scan_status = {**INIT, ...}` at every stage
transition. It REPLACES the dict; it does not update it in place. So nothing may hold the
dict by value — a reference captured once goes permanently stale at the first stage change,
and would report "listing" forever while the scan ran to completion.

That is why this module exports `status()`, a getter, and why appstate publishes
`scan_status` as a CALLABLE rather than a dict. appstate.py already says so in a comment;
this is the code that makes it true.

━━━ AND WHY THE SERVER ROOT IS READ, NEVER DERIVED ━━━

`_background_scan` seeds the builtin content, which needs the directory holding server.py.
`Path(__file__).resolve().parent` is correct in server.py and silently WRONG here (it
yields lib/, which has no docs/ or data/) — and it fails by finding nothing rather than by
raising, so the seeds would just quietly never run. server.py publishes the root once, as
appstate.server_root. Read it; never re-derive it.
"""
import concurrent.futures
import logging
import multiprocessing
import os
import sys
import threading
from pathlib import Path

import appstate
import builtin_content
import enrichment
import loosefolder as loosefolder_mod
import sloppak as sloppak_mod
from appconfig import _load_config
from dlc_paths import _get_dlc_dir
from env_compat import getenv_compat
from scan_worker import _relpath, _scan_one

log = logging.getLogger("feedBack.scan")

import json


# ── Directory-signature fast path ─────────────────────────────────────────────
#
# A startup scan globs the whole library twice (*.feedpak, *.wem) and stats every
# file to detect what changed. On a 50k-song library that lives on a slow mount
# (an NTFS-3G FUSE volume here) it is ~100k filesystem round trips every launch —
# the "big drive churns on every startup" report.
#
# But adds / removes / renames of songs all bump the mtime of the DIRECTORY that
# holds them (verified on the target NTFS-3G mount), and so does the addition of
# a subdirectory (a new entry in its parent). So after a scan we record every
# library directory and its mtime; on the next scan we re-stat ONLY those
# directories (a handful, vs 100k file ops). If none changed, the file set is
# unchanged and the whole listing/stat pass is skipped.
#
# The one thing this cannot see is a file edited IN PLACE under the same name —
# that bumps the file's mtime but not its directory's. That is rare for a song
# library (you add and remove packs, you don't rewrite them under the same name),
# and the manual Refresh forces a full scan (force=True) for exactly that case.
def _dir_signature_file() -> Path:
    return appstate.config_dir / "scan_dir_signature.json"


def _load_dir_signature() -> dict | None:
    try:
        data = json.loads(_dir_signature_file().read_text(encoding="utf-8"))
        if isinstance(data, dict) and isinstance(data.get("dirs"), dict):
            return data
    except (OSError, ValueError):
        pass
    return None


def _save_dir_signature(dlc: Path, dirs: dict[str, int]) -> None:
    # Keyed by the DLC path so switching libraries never matches a stale
    # signature. Best-effort: a failed write just means the next scan is a full
    # one, never a wrong one.
    try:
        _dir_signature_file().write_text(
            json.dumps({"dlc": str(dlc), "dirs": dirs}), encoding="utf-8")
    except OSError as e:
        log.debug("scan: could not persist dir signature: %s", e)


def _library_dirs(all_songs, dlc: Path) -> set[str]:
    """Every directory whose mtime reflects an add/remove of a library song:
    each song's containing directory and all of its ancestors up to the DLC
    root (the root itself always included, as "."). Derived from the already-
    listed songs — no extra filesystem walk. The builtin carve-outs
    (tutorials-builtin / minigames-builtin) are absent because the caller
    already excluded them from `all_songs`, so a minigame writing a drill there
    never invalidates the fast path.

    Directory-form songs (loose-song folders, directory sloppak bundles) also
    record their OWN directory: a file added/removed/replaced INSIDE the folder
    bumps that folder's mtime but not its parent's, so tracking only the parent
    would miss an in-place change to such a song. File-form sloppaks (a single
    .feedpak zip) aren't dirs, so they add nothing here — the flat file library
    stays at a handful of dir stats."""
    rels = {"."}
    for f in all_songs:
        rel = Path(_relpath(f, dlc))
        if f.is_dir():
            rels.add(rel.as_posix())
        parent = rel.parent
        rels.add(parent.as_posix())
        for anc in parent.parents:
            rels.add(anc.as_posix())
    return rels


def _has_unextracted_columns() -> bool:
    """True while any `songs` row still carries NULL in a column added by an
    additive migration — i.e. metadata the current extractor would fill but
    that no existing row has yet (currently `bass_tuning_name`).

    The tree-signature fast path only asks "did the file set change"; on a
    settled library the answer is no forever, so a schema addition would never
    reach extraction. This one-row probe forces the full pass exactly until the
    backfill completes — `put()` writes '' rather than NULL, so it self-clears
    after the rescan instead of disabling the fast path permanently."""
    try:
        from metadata_db import MetadataDB
        cond = " OR ".join(f"{c} IS NULL" for c in MetadataDB._EXTRACTION_MARKER_COLS)
        row = appstate.meta_db.conn.execute(
            f"SELECT 1 FROM songs WHERE {cond} LIMIT 1").fetchone()
    except Exception as e:
        # A probe failure must not take the scan down; falling back to the fast
        # path costs at most a delayed backfill.
        log.debug("scan: unextracted-column probe failed: %s", e)
        return False
    return row is not None


def _record_dir_signature(all_songs, dlc: Path) -> None:
    sig = _stat_dirs(dlc, _library_dirs(all_songs, dlc))
    if sig is not None:   # a dir vanished mid-scan → skip; next scan is full
        _save_dir_signature(dlc, sig)


def _stat_dirs(dlc: Path, rels) -> dict[str, int] | None:
    """{reldir: mtime_ns} for the given library dirs, or None if any is gone or
    unreadable — a vanished recorded dir means the tree changed, so fail to a
    full scan rather than a false match."""
    out: dict[str, int] = {}
    for rel in rels:
        try:
            out[rel] = (dlc if rel == "." else dlc / rel).stat().st_mtime_ns
        except OSError:
            return None
    return out


_SCAN_STATUS_INIT = {"running": False, "stage": "idle", "total": 0, "done": 0, "current": "", "error": None, "is_first_scan": False, "added": 0, "removed": 0}


_scan_status = dict(_SCAN_STATUS_INIT)

# Mass-prune guard thresholds for automatic scans (not full rescan).
# Refuse when would_remove >= max(_PRUNE_MAX_ABS, _PRUNE_MAX_FRAC * existing).
# At 50 % of the library (or even a single row on tiny libraries) a sudden
# disappearance almost certainly means a degraded mount, not a real deletion.
_PRUNE_MAX_ABS = 1
_PRUNE_MAX_FRAC = 0.5


def _make_scan_executor():
    """Build the executor for the background metadata scan.

    A `spawn` ProcessPoolExecutor in production. `spawn` (not the platform
    default) is mandatory: _background_scan runs on a non-main daemon
    thread, and forking a multithreaded process from a non-main thread can
    deadlock on locks held by other threads at fork time (the default on
    Linux). `spawn` boots a clean interpreter that imports only scan_worker
    (+ its pure lib deps) to unpickle the worker — never this module — so
    workers don't re-run server.py's import-time side effects (reopening
    SQLite, attaching a second RotatingFileHandler, re-registering routes).

    Tests monkeypatch this to a ThreadPoolExecutor so the scan runs
    in-process and metadata extraction can be mocked.
    """
    mp_ctx = multiprocessing.get_context("spawn")
    # Default to one worker per core so CPU-bound metadata parsing uses the
    # whole machine (the point of moving to processes).
    # FEEDBACK_MAX_SCAN_WORKERS (set by the Desktop launcher to cap memory
    # usage on low-RAM machines — e.g. 8 GB M2 MacBook Air) takes priority;
    # SCAN_MAX_WORKERS is a legacy override for Docker/bare installs.
    # A malformed override falls back to the core count rather than crashing.
    try:
        max_workers = int(
            getenv_compat("FEEDBACK_MAX_SCAN_WORKERS")
            or os.environ.get("SCAN_MAX_WORKERS")
            or (os.cpu_count() or 1)
        )
    except ValueError:
        max_workers = os.cpu_count() or 1
    # ProcessPoolExecutor raises ValueError on Windows when max_workers > 61
    # (the WaitForMultipleObjects handle limit), so clamp there — otherwise
    # a high-core Windows host can't construct the pool and the scan never
    # starts.
    if sys.platform == "win32":
        max_workers = min(max_workers, 61)
    return concurrent.futures.ProcessPoolExecutor(
        max_workers=max(1, max_workers), mp_context=mp_ctx,
    )


def background_scan(force: bool = False, allow_mass_prune: bool = False):
    """Scan the library and cache song metadata on startup. Uses a process pool to bypass the GIL for CPU-bound metadata parsing.

    `force` skips the directory-signature fast path and always does the full
    listing/stat pass — the manual Refresh sets it (see _dir_signature_file).

    `allow_mass_prune` permits the scan to prune more than the catastrophic
    threshold (_PRUNE_MAX_FRAC of the library). Only set by /api/rescan/full
    (explicit user intent); automatic and plain /api/rescan scans leave it
    False so a degraded-mount partial listing can't silently wipe the library.

    Never sets `_scan_status["running"] = False` — ownership of that flag
    lives in `_scan_runner` so a `kick_scan()` racing this function's
    terminal write cannot observe a stale False and start a second runner.
    """
    global _scan_status
    _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "listing"}

    # Load config once so both the DLC-dir lookup and the platform filter
    # read from the same snapshot, avoiding a redundant parse of config.json.
    _cfg = _load_config(appstate.config_dir / "config.json") or appstate.default_settings()
    dlc = _get_dlc_dir(_cfg)
    if not dlc:
        _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "idle", "error": "DLC folder not configured"}
        log.warning("Scan: no DLC folder configured")
        return

    builtin_content.seed_builtin_diagnostic_sloppaks(appstate.server_root, dlc)
    builtin_content.seed_builtin_starter_content(appstate.server_root, dlc)

    # Fast path: if every library directory recorded by the last scan still has
    # the same mtime, nothing was added, removed, or renamed, so the whole
    # glob-and-stat pass below can be skipped (see the signature comment above).
    # `force` (manual Refresh) always does the full pass. Seeding above is
    # idempotent — it only writes when a builtin is missing — so it does not
    # perturb the mtimes on a settled library.
    if not force and not _has_unextracted_columns():
        stored = _load_dir_signature()
        if stored is not None and stored.get("dlc") == str(dlc):
            current = _stat_dirs(dlc, stored["dirs"].keys())
            if current is not None and current == stored["dirs"]:
                _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "complete"}
                log.info("Scan: library tree unchanged (%d dirs) — skipped the full listing/stat pass",
                         len(current))
                return

    # Listing can fail on macOS without Full Disk Access, or on Docker if the
    # path isn't shared. Report the failure explicitly rather than silently
    # appearing to scan nothing.
    try:
        # Generated-content sloppaks that the highway WS must resolve by path
        # but that are NOT library songs. Two conventions share this carve-out:
        #   - tutorials-builtin/  — lesson drills seeded by the tutorials plugin
        #     (see plugins/tutorials/routes.py::_seed_builtin_packs).
        #   - minigames-builtin/  — exercise charts generated on demand by
        #     minigame plugins (e.g. Chord Sprint writes alternating-chord
        #     drills here). Cached/reused per exercise, never browsed.
        # Both are kept out of the scan; _resolve_dlc_path still loads them by
        # path for playback.
        def _is_excluded_from_library(p: Path) -> bool:
            return "tutorials-builtin" in p.parts or "minigames-builtin" in p.parts
        # Sloppaks: match both file (zip) and directory form, across both the
        # `.feedpak` and legacy `.sloppak` suffixes.
        _cands = sorted(p for ext in sloppak_mod.SONG_EXTS for p in dlc.rglob(f"*{ext}"))
        sloppaks = [f for f in _cands
                    if sloppak_mod.is_sloppak(f)
                    and not _is_excluded_from_library(f)]

        # Loose song folders: any directory containing a non-preview *.wem + *.xml.
        # Skip directories that are actually sloppak bundles — those are
        # already in `sloppaks`; the dispatcher's sloppak-first precedence
        # would route them to the sloppak path anyway, but adding them
        # here would inflate the scan queue and over-count the total.
        loose_songs = []
        seen_loose = set()
        sloppak_dirs = {p for p in sloppaks if p.is_dir()}
        for wem in sorted(dlc.rglob("*.wem")):
            if "preview" in wem.stem.lower():
                continue
            if _is_excluded_from_library(wem):
                continue
            d = wem.parent
            if d in sloppak_dirs or d.name.lower().endswith(sloppak_mod.SONG_EXTS):
                continue
            if d not in seen_loose and loosefolder_mod.is_loose_song(d):
                loose_songs.append(d)
                seen_loose.add(d)
    except PermissionError as e:
        msg = (f"Permission denied reading {dlc}. "
               "On macOS: grant Full Disk Access to the app in System Settings → Privacy & Security. "
               "With Docker: share this path in Docker Desktop → Settings → Resources → File Sharing.")
        log.error("Scan failed: %s (%s)", msg, e)
        _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "error", "error": msg}
        return
    except OSError as e:
        log.error("Scan failed listing %s: %s", dlc, e)
        _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "error", "error": f"Unable to list {dlc}: {e}"}
        return

    all_songs = sloppaks + loose_songs
    log.info("Scan: listed %d sloppaks and %d loose folders in %s",
             len(sloppaks), len(loose_songs), dlc)

    current_files = {_relpath(f, dlc) for f in all_songs}

    # Guard: refuse (or warn) when the listing suggests a degraded mount.
    # Two cases share the same logic:
    #   1. Zero listing  — current_files empty, DB non-empty: would erase everything.
    #   2. Partial listing — current_files non-empty but so many DB rows are absent
    #      that it looks like a mount glitch rather than deliberate deletions.
    #      Threshold: would_remove >= max(_PRUNE_MAX_ABS, _PRUNE_MAX_FRAC * existing).
    # allow_mass_prune (only True for /api/rescan/full) lets the prune proceed with
    # a warning so the user's explicit intent is honoured even in the degraded case.
    with appstate.meta_db._lock:
        _existing = appstate.meta_db.conn.execute(
            "SELECT COUNT(*) FROM songs").fetchone()[0]

    if _existing > 0:
        if not current_files:
            _would_remove = _existing
        else:
            with appstate.meta_db._lock:
                _db_files = {r[0] for r in appstate.meta_db.conn.execute(
                    "SELECT filename FROM songs").fetchall()}
            _would_remove = len(_db_files - current_files)

        _threshold = max(_PRUNE_MAX_ABS, _PRUNE_MAX_FRAC * _existing)
        if _would_remove >= _threshold:
            _msg = (
                f"Scan: would remove {_would_remove} of {_existing} DB rows "
                f"(threshold {int(_threshold)}) with only {len(current_files)} song(s) visible "
                "— possible mount/permission issue. Check the DLC mount; use Settings → "
                "Rescan Library (full) to authorise a large prune."
            )
            if allow_mass_prune:
                log.warning("%s — proceeding (user-authorised full rescan)", _msg)
            else:
                log.error("%s", _msg)
                _scan_status = {**_SCAN_STATUS_INIT, "running": True,
                                "stage": "error", "error": _msg}
                return

    # Clean up stale DB entries. delete_missing reports both deltas (rows pruned
    # + genuinely-new files) so the scan can surface an added/removed summary.
    _delta = appstate.meta_db.delete_missing(current_files)
    removed, added = _delta["removed"], _delta["added"]
    if removed:
        log.info("Removed %d stale DB entries", removed)

    # Figure out which need scanning
    to_scan = []
    for f in all_songs:
        # Skip entries that vanish or become unreadable between listing
        # and stat. Without this, one concurrent move/delete in DLC_DIR
        # would crash the scan thread and leave `_scan_status["running"]`
        # stuck true with no path to recover.
        try:
            mtime, size = appstate.stat_for_cache(f)
        except OSError as e:
            log.debug("scan: skipping %s (%s)", f, e)
            continue
        cache_key = _relpath(f, dlc)
        try:
            cached = appstate.meta_db.get(cache_key, mtime, size)
        except Exception as e:
            # Keep scanning even if a single metadata lookup fails.
            # The file will be re-scanned and cache repaired by put().
            log.warning("scan cache lookup failed for %s: %s", cache_key, e)
            cached = None
        if not cached:
            to_scan.append((f, mtime, size, dlc))
        elif any(cached.get(c) is None for c in appstate.meta_db._EXTRACTION_MARKER_COLS):
            # Row predates one of the per-perspective tuning columns (NULL
            # from the additive migration), so that perspective's tuning was
            # never extracted for it. Without this
            # re-queue an existing library would keep every bass column empty
            # forever — mtime/size still match, so nothing else would ever
            # bring the row back through extraction. Converges: put() always
            # writes '' (never NULL), so a rescanned row is never re-queued.
            to_scan.append((f, mtime, size, dlc))
        elif cached.get("arrangements") and any(
            "smart_name" not in a for a in cached["arrangements"]
        ):
            # Row was scanned before smart naming was introduced — force a
            # rescan so the DB picks up authoritative path flags from the
            # manifest JSON and stores correct smart_name values. Don't
            # re-queue rows where smart_name is explicitly null: the writer
            # only emits that when compute_smart_names truly can't classify
            # the arrangement (e.g. a name outside the recognised set with
            # zero path flags), so rescanning would produce the same null
            # forever and never converge.
            to_scan.append((f, mtime, size, dlc))

    if not to_scan:
        # Full pass completed with the DB already up to date — record the tree
        # signature so the next startup can take the fast path.
        _record_dir_signature(all_songs, dlc)
        _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "complete", "added": added, "removed": removed}
        log.info("Scan: nothing new to scan (%d songs, all cached)", len(all_songs))
        return

    # Refine: all discovered songs need scanning → treat as first-time import
    # (covers moved DLC folder / fully-stale DB as well as a genuinely empty DB).
    is_first_scan = bool(all_songs) and len(to_scan) == len(all_songs)
    _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "scanning", "total": len(to_scan),
                    "is_first_scan": is_first_scan}
    log.info("Library: %d sloppaks + %d loose folders, %d cached, %d to scan",
             len(sloppaks), len(loose_songs), len(all_songs) - len(to_scan), len(to_scan))

    with _make_scan_executor() as executor:
        futures = {executor.submit(_scan_one, item): item[0].name for item in to_scan}
        for future in concurrent.futures.as_completed(futures):
            fname = futures[future]
            try:
                name, mtime, size, meta = future.result()
                appstate.meta_db.put(name, mtime, size, meta)
            except Exception as e:
                log.warning("scan failed for %s: %s", fname, e)
            _scan_status["done"] += 1
            _scan_status["current"] = fname

    # Record the tree signature after a completed full pass so the next startup
    # can skip it when nothing has changed.
    _record_dir_signature(all_songs, dlc)
    log.info("Scan complete: %d songs cached", len(to_scan))
    _scan_status = {**_SCAN_STATUS_INIT, "running": True, "stage": "complete", "added": added, "removed": removed}


_scan_kick_lock = threading.Lock()


_scan_rescan_pending = False
# Set by kick_scan(force=True); consumed by _scan_runner for the next pass so a
# manual Refresh bypasses the directory-signature fast path.
_scan_force_next = False
# Set by kick_scan(allow_mass_prune=True); allows the next pass to prune past the
# catastrophic threshold. Sticky like _scan_force_next: if any queued request asks
# for it, the follow-up pass honours it.
_scan_mass_prune_next = False


# Handles to the running scan / enrichment worker threads. Both use the shared
# MetadataDB connection, so teardown/shutdown MUST join them before closing that
# connection — a daemon thread mid-query on a closed SQLite conn is a native
# use-after-free that segfaults the process (seen flaky in CI). Set by
# _kick_scan / _kick_enrich; joined by _join_background_db_threads().
_scan_thread: threading.Thread | None = None


def kick_scan(force: bool = False, allow_mass_prune: bool = False) -> bool:
    """Request a library rescan, single-flight + coalescing.

    `force` skips the directory-signature fast path for the resulting pass (the
    manual Refresh uses it so an in-place same-name edit — the one thing the
    fast path can't see — is always picked up). A forced request that coalesces
    onto a running or queued scan keeps the force intent: the pass is forced if
    ANY pending request asked for it.

    `allow_mass_prune` permits the resulting pass to prune past the catastrophic
    threshold. Sticky: if any pending request set it, the follow-up pass honours it.
    Only /api/rescan/full passes True — plain rescans and startup scans never do.

    Returns True if a new scan thread was started, False if one was already
    running. In the latter case a follow-up pass is queued and runs as soon
    as the current scan finishes so files landing mid-scan (e.g. an upload
    that finalizes after the scan has already listed DLC_DIR) are not lost
    until the next periodic pass. Multiple late-arriving requests coalesce
    into a single follow-up.
    """
    global _scan_rescan_pending, _scan_thread, _scan_force_next, _scan_mass_prune_next
    with _scan_kick_lock:
        if force:
            _scan_force_next = True
        if allow_mass_prune:
            _scan_mass_prune_next = True
        if _scan_status["running"]:
            _scan_rescan_pending = True
            return False
        # Mark running synchronously so a parallel kick_scan() observes it
        # before the worker thread has a chance to reassign _scan_status.
        _scan_status["running"] = True
    _scan_thread = threading.Thread(target=_scan_runner, daemon=True)
    _scan_thread.start()
    return True


def _scan_runner():
    """Run _background_scan, then re-run if requests arrived mid-scan."""
    global _scan_rescan_pending, _scan_force_next, _scan_mass_prune_next
    while True:
        # Consume both flags for THIS pass; requests queued mid-scan set them
        # again for the follow-up (sticky: any requester who asked for it wins).
        with _scan_kick_lock:
            forced = _scan_force_next
            _scan_force_next = False
            mass_prune = _scan_mass_prune_next
            _scan_mass_prune_next = False
        try:
            background_scan(force=forced, allow_mass_prune=mass_prune)
        except Exception:
            log.exception("background scan failed unexpectedly")

        with _scan_kick_lock:
            if not _scan_rescan_pending:
                _scan_status["running"] = False
                break
            _scan_rescan_pending = False
            _scan_status["running"] = True
    # Enrichment rides scan completion (library-metadata design §6): the scan
    # pool is a side-effect-free, no-network process pool by design, so
    # enrichment is a SEPARATE post-scan pass — non-blocking, the library is
    # usable immediately. The 5-minute periodic rescan re-kicks it, which is
    # the natural low-priority retry hook.
    enrichment._kick_enrich()


def status() -> dict:
    """The live scan status.

    A GETTER, deliberately. `_scan_status` is REBOUND on every stage transition, so a
    caller holding the dict would be reading a snapshot frozen at whatever stage it
    happened to grab — see the module header.
    """
    return _scan_status


def scan_thread():
    """The background scan thread, or None. Read by shutdown to join it."""
    return _scan_thread