Merge remote-tracking branch 'origin/main' into feat/mb-alias-scoring

# Conflicts:
#	tests/test_mb_enrichment.py
#	tests/test_mb_match.py

lib/mb_match.py

@@ -144,12 +144,31 @@ def _duration_int(v):
         return None
 
 
+def cand_artist_sim(song: dict, cand: dict) -> float:
+    """Best artist similarity between the song's reference artist and the
+    candidate's PRIMARY name OR any of its `artist_aliases` (romanized/alternate
+    names). MusicBrainz stores many artists under a non-Latin primary name
+    (大橋純子) with the romanized form ("Junko Ohashi") only as an alias, so a
+    reference typed/derived in romaji scores 0 against the primary but 1.0
+    against the alias. The caller (server) attaches `artist_aliases` only for
+    promising near-misses, so this is a plain max when they're present and the
+    original single comparison when they're not."""
+    best = similarity(song.get("artist"), cand.get("artist"), artist=True)
+    for alias in cand.get("artist_aliases") or []:
+        if best >= 1.0:
+            break
+        s = similarity(song.get("artist"), alias, artist=True)
+        if s > best:
+            best = s
+    return best
+
+
 def score_candidate(song: dict, cand: dict) -> float:
     """Combined confidence that MusicBrainz candidate `cand` is the song the
     chart transcribes. 0.5*artist + 0.5*title, plus small year/duration
     corroboration bonuses, capped at 1.0. Missing fields score 0 on their
     half — classify() separately refuses to auto-match without both."""
-    artist_sim = similarity(song.get("artist"), cand.get("artist"), artist=True)
+    artist_sim = cand_artist_sim(song, cand)
     title_sim = similarity(song.get("title"), cand.get("title"))
     score = 0.5 * artist_sim + 0.5 * title_sim
     sy, cy = _year_int(song.get("year")), _year_int(cand.get("year"))
@@ -180,7 +199,7 @@ def classify(song: dict, cand: dict, score: float, auto_min: float | None = None
     """
     if auto_min is None:
         auto_min = AUTO_MIN
-    artist_sim = similarity(song.get("artist"), cand.get("artist"), artist=True)
+    artist_sim = cand_artist_sim(song, cand)
     title_sim = similarity(song.get("title"), cand.get("title"))
     if (score >= auto_min and artist_sim >= AUTO_ARTIST_MIN
             and title_sim >= AUTO_TITLE_MIN):
@@ -234,12 +253,37 @@ def _lucene_escape_phrase(s: str) -> str:
 _LIVE_GROUP_RE = re.compile(r"[(\[][^)\]]*\blive\b[^)\]]*[)\]]", re.IGNORECASE)
 
 
-def build_recording_query(artist, title) -> str:
+def build_recording_query(artist, title, *, loose: bool = False) -> str:
     """Lucene query for /ws/2/recording. Built from the DENOISED fields —
     the noise we strip (author credits, "(Live)", "(v2)") would otherwise
-    poison the search server's own scoring."""
+    poison the search server's own scoring.
+
+    ``loose=True`` drops the field-scoped quoted PHRASES for plain AND-ed
+    term groups (``(telephone number) AND (junko ohashi)``). The point:
+    a field phrase like ``artist:"Junko Ohashi"`` only matches MusicBrainz's
+    *primary* artist name — it never searches ALIASES — so a recording stored
+    under a non-Latin primary (大橋純子) whose romanized name is only an alias
+    is invisible to the strict query. A loose term query searches the whole
+    document, aliases included, and surfaces it. Lower precision by design: it
+    is a FALLBACK for when the strict query returns nothing, and its results
+    are re-scored by ``rank_candidates`` (and, for auto-match, gated by the
+    per-field floors), so noise never auto-applies."""
     t = denoise(title)
     a = denoise(artist)
+    if loose:
+        # denoise() already reduced each field to lowercase [a-z0-9 and] tokens
+        # (punctuation → spaces, diacritics stripped, & → "and"), so no
+        # Lucene-special character survives to need escaping. Group each field's
+        # terms and require both groups.
+        q = " AND ".join("(%s)" % g for g in (t, a) if g)
+        # Keep the SAME live exclusion as the strict path: the loose query is
+        # lower-precision, and score_candidate doesn't penalize a live take, so
+        # without this a studio chart whose strict query missed could fall back
+        # to — and auto-confirm — a live-only recording. Skipped only when the
+        # source title is itself a live take (mirrors the strict path).
+        if q and not _LIVE_GROUP_RE.search(str(title or "")):
+            q += " AND -secondarytype:Live"
+        return q
     parts = []
     if t:
         parts.append('recording:"%s"' % _lucene_escape_phrase(t))

server.py

@@ -6320,15 +6320,30 @@ def _mb_http_get(path: str, params: dict) -> dict | None:
 
 
 def _mb_search_recordings(artist, title, limit: int = 12) -> list[dict]:
-    """Text search (tier 2–4): denoised Lucene query over /recording. The query
+    """Text search (tier 2–4): denoised Lucene query over /recording. The strict
-    now drops live-only recordings and our ranker rewards the studio take, so a
+    query drops live-only recordings and the ranker rewards the studio take, so a
     slightly larger default result set gives the re-ranker room to surface the
-    canonical version (one request per song regardless of limit)."""
+    canonical version.
+
+    Runs the strict field-phrase query first (high precision); if it finds
+    nothing, retries ONCE with a loose term query. The strict phrase only matches
+    MusicBrainz's *primary* artist/title, so a recording stored under a non-Latin
+    primary name (大橋純子) whose romanized form ("Junko Ohashi") is only an alias
+    is invisible to it — the loose query searches aliases and rescues it. The
+    retry spends a second throttled request only on a miss; results are re-scored
+    by rank_candidates, so the looser recall doesn't lower match quality
+    (auto-accept still needs the per-field floors)."""
     query = mb_match.build_recording_query(artist, title)
-    if not query:
+    cands: list[dict] = []
-        return []
+    if query:
-    body = _mb_http_get("recording", {"query": query, "limit": limit})
+        body = _mb_http_get("recording", {"query": query, "limit": limit})
-    return mb_match.parse_search_response(body or {})
+        cands = mb_match.parse_search_response(body or {})
+    if not cands:
+        loose = mb_match.build_recording_query(artist, title, loose=True)
+        if loose and loose != query:
+            body = _mb_http_get("recording", {"query": loose, "limit": limit})
+            cands = mb_match.parse_search_response(body or {})
+    return cands
 
 
 # ── AcoustID audio fingerprinting (content-based identification) ──────────────
@@ -6529,6 +6544,78 @@ _MBID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f
 _ISRC_RE = re.compile(r"^[A-Z]{2}[A-Z0-9]{3}[0-9]{7}$")
 
 
+# ── Alias-aware scoring ───────────────────────────────────────────────────────
+# MusicBrainz stores many artists under a non-Latin PRIMARY name (大橋純子) with
+# the romanized form ("Junko Ohashi") only as an ALIAS. A recording search
+# returns the primary name in its artist-credit, never the aliases — so scoring
+# a romanized reference against the primary gives 0 and the match can't confirm.
+# We fetch the artist's aliases (one throttled lookup, process-cached) and hand
+# them to the scorer, but ONLY for a promising near-miss (title already agrees,
+# artist doesn't) so a normal pass spends no extra requests.
+_ALIAS_ENRICH_MAX = 3          # cap alias lookups per song/search (each is ≤1/s)
+_artist_alias_cache: dict[str, list[str]] = {}
+
+
+def _mb_artist_aliases(artist_id: str) -> list[str]:
+    """Romanized/alternate names for a MusicBrainz artist, process-cached (an
+    artist recurs across a whole discography, so a library of one artist costs
+    ONE lookup). Returns [] for an unknown/aliasless artist. Raises
+    EnrichTransportError on a network failure so the caller pauses the pass
+    (nothing is cached on failure → retried next pass)."""
+    aid = str(artist_id or "")
+    if aid in _artist_alias_cache:
+        return _artist_alias_cache[aid]
+    if not _MBID_RE.match(aid):
+        return []
+    body = _mb_http_get(f"artist/{aid}", {"inc": "aliases"})
+    names: list[str] = []
+    if body:
+        sort_name = str(body.get("sort-name") or "").strip()
+        if sort_name:
+            names.append(sort_name)          # often the romanized form for JP artists
+        for al in body.get("aliases") or []:
+            if isinstance(al, dict) and al.get("name"):
+                names.append(str(al["name"]))
+    seen: set[str] = set()
+    out: list[str] = []
+    for n in names:
+        k = n.casefold()
+        if k and k not in seen:
+            seen.add(k)
+            out.append(n)
+    out = out[:12]
+    _artist_alias_cache[aid] = out
+    return out
+
+
+def _alias_enrich(ref: dict, cands: list[dict]) -> None:
+    """Attach `artist_aliases` in place to candidates that look like the
+    non-Latin-primary case — title agrees with the reference but the primary
+    artist doesn't — so the scorer can confirm them via a romanized alias.
+    Bounded by _ALIAS_ENRICH_MAX + the process cache; a no-op when the
+    reference has no artist or nothing is aliasable."""
+    ref_artist = (ref.get("artist") or "").strip()
+    if not ref_artist:
+        return
+    spent = 0
+    for c in cands:
+        if spent >= _ALIAS_ENRICH_MAX:
+            break
+        if not isinstance(c, dict) or c.get("artist_aliases") is not None:
+            continue
+        aid = c.get("artist_id")
+        if not aid:
+            continue
+        # Only spend a lookup on a promising near-miss: the title already
+        # matches, but the primary artist doesn't (that's the alias signature).
+        if mb_match.similarity(ref.get("title"), c.get("title")) < mb_match.AUTO_TITLE_MIN:
+            continue
+        if mb_match.similarity(ref_artist, c.get("artist"), artist=True) >= mb_match.AUTO_ARTIST_MIN:
+            continue
+        c["artist_aliases"] = _mb_artist_aliases(aid)   # cached; attach [] to avoid refetch
+        spent += 1
+
+
 def _manifest_exact_ids(filename: str) -> dict:
     """Optional `mbid`/`isrc` from the pack manifest — the spec's additive
     identity keys. Feature-detected: packs published before that spec
@@ -6941,7 +7028,13 @@ def _enrich_one(row: dict, auto_min: float | None = None, field_filter=None,
                                            cand=field_filter(cands[0]) if field_filter else cands[0])
             return
 
-    ranked = mb_match.rank_candidates(ref, _mb_search_recordings(ref.get("artist"), ref.get("title")))
+    cands = _mb_search_recordings(ref.get("artist"), ref.get("title"))
+    # Alias-enrich promising near-misses (title agrees, primary artist doesn't)
+    # so a non-Latin-primary artist can confirm via its romanized alias, then
+    # rank once with the aliases in hand. `ref` carries any filename-derived
+    # artist seed, so alias scoring runs against the searched identity.
+    _alias_enrich(ref, cands)
+    ranked = mb_match.rank_candidates(ref, cands)
     best = ranked[0] if ranked else None
     tier = mb_match.classify(ref, best, best["score"], auto_min=auto_min) if best else "none"
     if tier == "auto":
@@ -7836,6 +7929,13 @@ def api_enrichment_search(artist: str = "", title: str = "", limit: int = 8,
     if duration and duration > 0 and not ref.get("duration"):
         ref = dict(ref)
         ref["duration"] = duration
+    # Alias-enrich so a non-Latin-primary artist (大橋純子) ranks by its
+    # romanized alias against the typed query ("Junko Ohashi") instead of
+    # sinking to the bottom with a 0 artist score.
+    try:
+        _alias_enrich(ref, cands)
+    except EnrichTransportError:
+        pass   # aliases are a ranking nicety here; fall back to primary-name scoring
     return {"candidates": mb_match.rank_candidates(ref, cands)}
 
 

tests/test_mb_enrichment.py

@@ -97,6 +97,94 @@ def mb_doc(rid="rec-1", title="Thunderstruck", artist="AC/DC", artist_id="art-1"
     }
 
 
+# ── strict-then-loose search fallback ────────────────────────────────────────
+
+def test_search_falls_back_to_loose_when_strict_is_empty(server, monkeypatch):
+    """The strict field-phrase query misses a non-Latin-primary artist; the
+    loose retry (no field scoping) searches aliases and finds it."""
+    calls = []
+
+    def _routed(path, params):
+        q = params.get("query", "")
+        calls.append(q)
+        if q.startswith("recording:"):        # strict phrase → nothing
+            return {"recordings": []}
+        return {"recordings": [mb_doc(rid="rec-x", title="Telephone Number")]}
+
+    monkeypatch.setattr(server, "_mb_http_get", _routed)
+    cands = server._mb_search_recordings("Junko Ohashi", "Telephone Number")
+    assert len(cands) == 1
+    assert len(calls) == 2                     # strict first, then the loose retry
+    assert calls[0].startswith("recording:")   # strict is the field-phrase form
+    assert "artist:" not in calls[1] and '"' not in calls[1]   # loose retry
+
+
+def test_search_does_not_retry_when_strict_hits(server, monkeypatch):
+    """A strict hit must not spend a second (throttled) request on the loose
+    query."""
+    calls = []
+
+    def _routed(path, params):
+        calls.append(params.get("query", ""))
+        return {"recordings": [mb_doc()]}
+
+    monkeypatch.setattr(server, "_mb_http_get", _routed)
+    cands = server._mb_search_recordings("AC/DC", "Thunderstruck")
+    assert len(cands) == 1
+    assert len(calls) == 1
+
+
+# ── alias-aware scoring (non-Latin-primary artists) ──────────────────────────
+
+_AID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+
+
+def test_artist_aliases_fetched_and_cached(server, monkeypatch):
+    calls = []
+
+    def fake(path, params):
+        calls.append(path)
+        return {"sort-name": "Ohashi, Junko",
+                "aliases": [{"name": "Junko Ohashi"}, {"name": "大橋 純子"}]}
+
+    monkeypatch.setattr(server, "_mb_http_get", fake)
+    names = server._mb_artist_aliases(_AID)
+    assert "Junko Ohashi" in names and "Ohashi, Junko" in names
+    server._mb_artist_aliases(_AID)          # cached → no second request
+    assert len(calls) == 1
+
+
+def test_artist_aliases_rejects_bad_id(server, monkeypatch):
+    def boom(path, params):
+        raise AssertionError("must not fetch for a non-UUID id")
+
+    monkeypatch.setattr(server, "_mb_http_get", boom)
+    assert server._mb_artist_aliases("not-a-uuid") == []
+
+
+def test_enrich_auto_matches_japanese_primary_via_alias(server, monkeypatch):
+    # A pack whose (romanized) artist MB stores under a Japanese primary name.
+    _put(server, "x.sloppak", title="Telephone Number", artist="Junko Ohashi")
+
+    def _routed(path, params):
+        if path.startswith("artist/"):                      # alias lookup
+            return {"sort-name": "Ohashi, Junko",
+                    "aliases": [{"name": "Junko Ohashi"}]}
+        q = params.get("query", "")
+        if q.startswith("recording:"):                      # strict phrase → nothing
+            return {"recordings": []}
+        return {"recordings": [mb_doc(rid="rec-jp", title="Telephone Number",
+                                      artist="大橋純子", artist_id=_AID)]}   # loose hit
+
+    monkeypatch.setattr(server, "_mb_http_get", _routed)
+    monkeypatch.setattr(server, "_enrich_network_enabled", lambda: True)
+    server._background_enrich()
+    row = server.meta_db.get_enrichment("x.sloppak")
+    # The romanized alias lifts the artist over the auto floor → auto-confirmed.
+    assert row["match_state"] == "matched"
+    assert row["mb_recording_id"] == "rec-jp"
+
+
 # ── offline safety (the pytest-never-hits-network contract) ──────────────────
 
 def test_offline_default_skips_matching(server, monkeypatch):

tests/test_mb_match.py

@@ -189,6 +189,54 @@ def test_build_recording_query_escapes_and_handles_missing_artist():
     assert "artist:" not in q
 
 
+def test_build_recording_query_loose_drops_field_phrases():
+    # The strict form locks to the *primary* artist/title phrase (and drops
+    # live-only recordings — the chart isn't a live take).
+    assert m.build_recording_query("Junko Ohashi", "Telephone Number") == \
+        'recording:"telephone number" AND artist:"junko ohashi" AND -secondarytype:Live'
+    # The loose form has no field scoping and no phrases, so MusicBrainz also
+    # searches artist ALIASES — rescues non-Latin-primary artists (大橋純子) —
+    # but keeps the same live exclusion (a studio chart must not fall back to a
+    # live-only recording).
+    loose = m.build_recording_query("Junko Ohashi", "Telephone Number", loose=True)
+    assert loose == "(telephone number) AND (junko ohashi) AND -secondarytype:Live"
+    assert "artist:" not in loose and '"' not in loose
+
+
+def test_build_recording_query_loose_missing_artist():
+    assert m.build_recording_query("", "Fantasy", loose=True) == \
+        "(fantasy) AND -secondarytype:Live"
+
+
+def test_build_recording_query_loose_keeps_live_for_live_charts():
+    # A live chart's loose fallback must NOT exclude live recordings (same gate
+    # as the strict path) — else its only correct recording is filtered out.
+    loose = m.build_recording_query("AC/DC", "Highway to Hell (Live at Donington)", loose=True)
+    assert "-secondarytype:Live" not in loose
+    assert loose == "(highway to hell) AND (ac dc)"
+
+
+# ── alias-aware artist scoring ────────────────────────────────────────────────
+
+def test_cand_artist_sim_uses_aliases():
+    song = {"artist": "Junko Ohashi", "title": "Telephone Number"}
+    # primary is the Japanese name → romanized reference scores 0…
+    assert m.cand_artist_sim(song, {"artist": "大橋純子"}) == 0.0
+    # …but a romanized alias confirms it
+    assert m.cand_artist_sim(
+        song, {"artist": "大橋純子", "artist_aliases": ["Ohashi Junko", "Junko Ohashi"]}) == 1.0
+
+
+def test_alias_lifts_candidate_to_auto():
+    song = {"artist": "Junko Ohashi", "title": "Telephone Number"}
+    jp = {"artist": "大橋純子", "title": "Telephone Number"}
+    # Without the alias: title matches but the artist floor fails → never auto.
+    assert m.classify(song, jp, m.score_candidate(song, jp)) != "auto"
+    # With the romanized alias attached: artist clears the floor → auto.
+    jp_alias = dict(jp, artist_aliases=["Junko Ohashi"])
+    assert m.classify(song, jp_alias, m.score_candidate(song, jp_alias)) == "auto"
+
+
 # ── MusicBrainz response parsing ──────────────────────────────────────────────
 
 MB_DOC = {