retrieval: remove the cross-encoder reranker (ablation showed no earned keep) (#100)

The 2026-07-10 built-in ablation on real content measured the reranker at
recall@8 0.810 (ON) vs 0.790 (OFF) and MRR 0.615 vs 0.620 — +0.02 recall,
-0.005 MRR. That does not justify a ~90MB cross-encoder loaded on the query
path (its cold download was the 17.9s latency outlier) plus ~0.3-0.5s per
query. On real content the dense+keyword RRF fusion already ranks well; the
reranker was a crutch from the empty-stub era. So it is removed, not just
disabled.

- index/retrieve.py: drop the rerank path and the rerank_fn param; the
interleaved per-kind RRF top-k is the final order (the exact-symbol pin
still applies). index/rerank.py deleted.
- app/main.py: drop the reranker warmup (no 90MB model to pre-load).
- eval.yml: drop the input and TORCHDOCS_RERANK env; run_retrieval
reverts from the ON/OFF ablation to a single pass (nothing left to ablate).
- conftest.py: drop the test-suite TORCHDOCS_RERANK=0 default; test_rerank.py
deleted. Stale comments in route.py / diagnose_retrieval.py updated.

218 tests pass; ruff clean.


Claude-Session: https://claude.ai/code/session_01LX1gm8fuzK2ZeEWi3Zar1b

Co-authored-by: Claude <noreply@anthropic.com>

.github/workflows/eval.yml

@@ -32,15 +32,9 @@ on:
           live index was built with, or the query vector width won't fit."
         type: string
         default: "BAAI/bge-small-en-v1.5"
-      rerank:
-        description: "cross-encoder rerank stage (the production default is ON;
-          pick 0 for a before/after ablation against the raw fused order)"
-        type: choice
-        options: ["1", "0"]
-        default: "1"
       k:
         description: "top-k sections retrieved per question (the app's default
-          is 8; try 4 to measure precision-over-volume after reranking)"
+          is 8; try 4 to measure precision-over-volume)"
         type: string
         default: "8"
       provider:
@@ -92,7 +86,6 @@ jobs:
           NEON_URL: ${{ secrets.NEON }}
           TORCHDOCS_EVAL_SET: ${{ inputs.set }}
           TORCHDOCS_EMBED_MODEL: ${{ inputs.model }}
-          TORCHDOCS_RERANK: ${{ inputs.rerank }}
           TORCHDOCS_RETRIEVAL_K: ${{ inputs.k }}
         run: python -m eval.run_retrieval
       - name: Rank diagnostic (why do the known misses miss)
@@ -175,7 +168,6 @@ jobs:
           TORCHDOCS_OPENAI_COMPAT_MODEL: ${{ inputs.llm }}
           TORCHDOCS_EVAL_SET: ${{ inputs.set }}
           TORCHDOCS_JUDGE_LIMIT: ${{ inputs.judge_limit }}
-          TORCHDOCS_RERANK: ${{ inputs.rerank }}
           TORCHDOCS_RETRIEVAL_K: ${{ inputs.k }}
         run: python -m eval.run_judge
       - name: Commit results back to the branch

agent/route.py

@@ -6,8 +6,8 @@ wall-clock — unacceptable for the common case. But the loop's value is real
 only for MULTI-SOURCE questions (catalog / compare / recipe / internals),
 where it assembles pages one search can't: measured agentic coverage 0.567 vs
 0.133 single-shot. A usage question ("how do I use SGD with momentum?") is
-answered by ONE retrieval pass + ONE generation — now backed by the reranked
-retrieval (recall@8 0.84).
+answered by ONE retrieval pass + ONE generation — backed by the hybrid
+per-kind RRF retrieval (recall@8 ~0.81 on real content).
 
 So: a zero-LLM-call heuristic sends multi-source shapes to the loop and
 everything else to the grounded single-shot path. The heuristic is allowed to

app/main.py

@@ -108,8 +108,7 @@ def _warm_up() -> None:
     """Load the embedding model once so the first question isn't slow.
 
     This also covers the guard: its topicality check embeds the (translated)
-    question with the same model. The reranker (when enabled) is warmed for
-    the same reason — its ~90MB cross-encoder must not download mid-question.
+    question with the same model.
     """
     try:
         from index.embed import embed_query
@@ -117,13 +116,6 @@ def _warm_up() -> None:
         embed_query("warmup")
     except Exception as exc:  # noqa: BLE001 — warmup is best-effort
         print(f"[app] warmup skipped: {exc}")
-    try:
-        from index import rerank
-
-        if rerank.enabled():
-            rerank.rerank("warmup", [{"url": ""}, {"url": ""}], k=1)
-    except Exception as exc:  # noqa: BLE001 — warmup is best-effort
-        print(f"[app] rerank warmup skipped: {exc}")
 
 
 def render(answer: Answer) -> str:

eval/diagnose_retrieval.py

@@ -85,7 +85,7 @@ def main() -> int:
             # The decisive number: the expected page's OWN nearest chunk — its
             # absolute cosine distance, and its true dense rank among ALL api
             # chunks (count of api chunks strictly closer). rank ~25 = a crowding
-            # problem (a deeper pool / rerank helps); a far distance / rank in the
+            # problem (a deeper pool helps); a far distance / rank in the
             # hundreds = an embedding problem (only doc-side enrichment helps).
             pat = f"%{expected}%"
             best = conn.execute(

eval/run_retrieval.py

@@ -31,14 +31,9 @@ from dotenv import load_dotenv
 
 EVAL_DIR = Path(__file__).parent
 EVAL_SET = os.environ.get("TORCHDOCS_EVAL_SET", "v1")
+# a k≠8 sweep suffixes the file so it can't masquerade as the standard k=8 run
 _K = os.environ.get("TORCHDOCS_RETRIEVAL_K", "8")
-
-
-def _results_path(rerank_on: bool) -> Path:
-    """Standard file for the reranked run; a _norerank sibling for the ablation
-    (a k≠8 sweep suffixes k too) — so neither masquerades as the other."""
-    variant = ("" if _K == "8" else f"_k{_K}") + ("" if rerank_on else "_norerank")
-    return EVAL_DIR / "results" / f"retrieval_{EVAL_SET}{variant}.jsonl"
+RESULTS = EVAL_DIR / "results" / f"retrieval_{EVAL_SET}{'' if _K == '8' else f'_k{_K}'}.jsonl"
 
 
 def load_questions() -> dict[str, dict]:
@@ -86,19 +81,16 @@ def question_metrics(expected: list[list[str]], pointers: list[dict]) -> dict:
     }
 
 
-def _measure(questions: dict[str, dict], k: int, *, rerank_on: bool) -> dict:
-    """One retrieval pass over the set at a fixed rerank setting.
-
-    The rerank on/off toggle is TORCHDOCS_RERANK, which retrieve() reads (via
-    index.rerank.enabled()) fresh on every call — so flipping the env here runs
-    the exact production path with the reranker in or out. Returns the aggregate
-    plus per-question records (also written to the pass's own results file)."""
-    os.environ["TORCHDOCS_RERANK"] = "1" if rerank_on else "0"
+def main() -> int:
+    load_dotenv()
     from index.retrieve import retrieve
 
-    label = "rerank ON " if rerank_on else "rerank OFF"
+    k = int(os.environ.get("TORCHDOCS_RETRIEVAL_K", "8"))
+    questions = load_questions()
+
+    RESULTS.parent.mkdir(exist_ok=True)
     records, recalls, mrrs = [], [], []
-    print(f"\n── {label} ──  eval set: {EVAL_SET} ({len(questions)} questions), k={k}")
+    print(f"eval set: {EVAL_SET}  ({len(questions)} questions), k={k}")
     print(f"{'id':<6}{'recall@' + str(k):<12}{'MRR':<8}misses")
     for qid, q in questions.items():
         expected = q.get("expected", [])
@@ -117,41 +109,17 @@ def _measure(questions: dict[str, dict], k: int, *, rerank_on: bool) -> dict:
             {"id": qid, "question": q["question"], "k": k, **m,
              "retrieved": [p["url"] + "#" + p.get("anchor", "") for p in pointers]}
         )
-    path = _results_path(rerank_on)
-    path.parent.mkdir(exist_ok=True)
-    with path.open("w") as out:
-        for record in records:
-            out.write(json.dumps(record, ensure_ascii=False) + "\n")
-    n = len(recalls)
-    return {
-        "recall": sum(recalls) / n if n else 0.0,
-        "mrr": sum(mrrs) / n if n else 0.0,
-        "n": n,
-        "path": path,
-    }
-
-
-def main() -> int:
-    load_dotenv()
-    k = int(os.environ.get("TORCHDOCS_RETRIEVAL_K", "8"))
-    questions = load_questions()
 
-    # measure BOTH configurations every run — the reranker's contribution is the
-    # ablation the team keeps asking for, so make it a permanent side-by-side
-    # instead of a manual rerank=0 dispatch nobody can trigger from mobile.
-    on = _measure(questions, k, rerank_on=True)
-    off = _measure(questions, k, rerank_on=False)
-    if not on["n"]:
+    if not recalls:
         print("no questions with expectations — nothing measured")
         return 1
-
-    d_recall = on["recall"] - off["recall"]
-    d_mrr = on["mrr"] - off["mrr"]
-    print(f"\n{'config':<14}{'recall@' + str(k):<12}{'MRR':<8}")
-    print(f"{'rerank ON':<14}{on['recall']:<12.3f}{on['mrr']:<8.3f}")
-    print(f"{'rerank OFF':<14}{off['recall']:<12.3f}{off['mrr']:<8.3f}")
-    print(f"{'Δ (rerank)':<14}{d_recall:<+12.3f}{d_mrr:<+8.3f}")
-    print(f"\nresults → {on['path']}  |  {off['path']}")
+    print(f"\naggregate over {len(recalls)} questions: "
+          f"mean recall@{k}={sum(recalls) / len(recalls):.3f}  "
+          f"mean MRR={sum(mrrs) / len(mrrs):.3f}")
+    with RESULTS.open("w") as out:
+        for record in records:
+            out.write(json.dumps(record, ensure_ascii=False) + "\n")
+    print(f"results → {RESULTS}")
     return 0
 
 

index/rerank.py

@@ -1,104 +0,0 @@
-"""Cross-encoder reranking: the precision stage after candidate generation.
-
-Why a reranker, and why now: the gloss re-embed (measured 2026-07-09) moved
-several expected pages INTO their kind-pool (LayerNorm: true dense rank
-899 → 2; SGD: 7 → 3) but bi-encoder distance + RRF still decide the final
-order. A cross-encoder reads query and candidate TOGETHER — the interaction a
-bi-encoder cannot see — and fixes exactly that ordering. It cannot rescue a
-page that never became a candidate (Linear at dense rank 3,412 needs deeper
-index-side enrichment, not reranking); see docs/retrieval-gaps-and-improvements.md.
-
-What the cross-encoder scores: the index stores no chunk content (pointers
-only), and hydrating dozens of candidates per query is too slow for the live
-app — so each candidate is scored on its metadata line: symbol + page title +
-heading path + the page's committed gloss (index/glosses.jsonl). For api pages
-the gloss is exactly the plain-language "what is this page for" sentence, which
-is the signal a relevance judgment needs.
-
-Runs on CPU (~90MB model, a few hundred ms for a 24-candidate slate), behind
-the TORCHDOCS_RERANK kill switch so eval can measure before/after and a broken
-model can be turned off without a deploy. Fail-open: any model error returns
-the fused order unchanged — reranking must never take retrieval down.
-"""
-
-from __future__ import annotations
-
-import os
-import threading
-from typing import Any
-
-RERANK_MODEL = os.environ.get("TORCHDOCS_RERANK_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2")
-
-# candidates fed to the cross-encoder: k=8 finals from a 24-candidate slate
-# keeps 3× headroom for the reorder while staying sub-second on CPU
-RERANK_SLATE = int(os.environ.get("TORCHDOCS_RERANK_SLATE", "24"))
-
-
-def enabled() -> bool:
-    """Kill switch, read per call so tests/deploys can flip it late."""
-    return os.environ.get("TORCHDOCS_RERANK", "1") not in ("0", "false", "no")
-
-
-_MODEL_LOCK = threading.Lock()
-_MODEL = None
-
-
-def _model():
-    # same double-checked load as index/embed._model: exactly one instance,
-    # concurrent first calls must not download/build the model twice
-    global _MODEL
-    if _MODEL is None:
-        with _MODEL_LOCK:
-            if _MODEL is None:
-                from sentence_transformers import CrossEncoder
-
-                print(f"[rerank] loading {RERANK_MODEL} (first run downloads ~90MB)")
-                _MODEL = CrossEncoder(RERANK_MODEL, device="cpu")
-    return _MODEL
-
-
-def _score_pairs(pairs: list[tuple[str, str]]) -> list[float]:
-    return [float(s) for s in _model().predict(pairs, show_progress_bar=False)]
-
-
-def rerank_text(pointer: dict[str, Any], glosses: dict[str, str]) -> str:
-    """The candidate line the cross-encoder judges the query against."""
-    from index.embed import symbol_from_url
-
-    parts = [
-        p
-        for p in (
-            symbol_from_url(pointer.get("url", "")),
-            pointer.get("page_title", ""),
-            pointer.get("heading_path", ""),
-            glosses.get(pointer.get("url", ""), ""),
-        )
-        if p
-    ]
-    return ". ".join(parts)
-
-
-def rerank(
-    query: str,
-    pointers: list[dict[str, Any]],
-    k: int,
-    scorer=None,
-) -> list[dict[str, Any]]:
-    """Reorder candidate pointers by cross-encoder relevance; return the top-k.
-
-    `scorer` (tests) replaces the model: (list of (query, text) pairs) → scores.
-    Fail-open: any scoring error keeps the incoming (RRF-fused) order.
-    """
-    if len(pointers) <= 1:
-        return pointers[:k]
-    from index.embed import load_glosses
-
-    glosses = load_glosses()
-    pairs = [(query, rerank_text(p, glosses)) for p in pointers]
-    try:
-        scores = (scorer or _score_pairs)(pairs)
-    except Exception as exc:  # noqa: BLE001 — reranking must never break retrieval
-        print(f"[rerank] scoring failed ({type(exc).__name__}: {exc}); keeping fused order")
-        return pointers[:k]
-    order = sorted(range(len(pointers)), key=lambda i: scores[i], reverse=True)
-    return [pointers[i] for i in order[:k]]

index/retrieve.py

@@ -8,12 +8,13 @@ interleaved strongest-first into the top-k, and the answering model judges
 what is actually relevant.
 This replaced a global-RRF design whose crowding fixes (a reference channel,
 reserved seats, distance gates) traded one benchmark regression for another.
-A cross-encoder rerank (index/rerank.py) then reorders a wide slate of the
-fused candidates into the final top-k — precision after candidate generation.
+(A cross-encoder rerank stage once reordered the fused slate; the 2026-07-10
+ablation on real content measured it at +0.02 recall / −0.005 MRR and it was
+removed — the fused RRF order is the final order.)
 
-Returns POINTERS (url, anchor, heading path, ...) — never content. The
-caller hydrates content from the snapshot (index/hydrate.py). This is the
-engine behind the agent's `search_docs` tool.
+Returns POINTERS (url, anchor, heading path, content, ...). The caller reads
+each pointer's stored content directly (index/hydrate.py). This is the engine
+behind the agent's `search_docs` tool.
 """
 
 from __future__ import annotations
@@ -157,7 +158,6 @@ def retrieve(
     pool: int = 20,
     debug: bool = False,
     kind: str | None = None,
-    rerank_fn=None,
 ) -> list[dict[str, Any]]:
     """Top-k pointers for a query, drawn from per-kind pools.
 
@@ -167,16 +167,16 @@ def retrieve(
     (e.g. `scaled_dot_product_attention`) that dense similarity misses.
 
     Candidates farther than RELEVANCE_GAP from the best hit are dropped, then
-    the pools are interleaved (strongest pool first) into a wide slate. A
-    cross-encoder rerank (index/rerank.py — the precision stage; kill switch
-    TORCHDOCS_RERANK) reorders the slate into the top-k; when it is off the
-    slate's first k are the top-k unchanged. An exact-symbol match is pinned
-    first, docs-search style.
+    the pools are interleaved (strongest pool first) and the first k are the
+    result. An exact-symbol match is pinned first, docs-search style. (A
+    cross-encoder rerank stage sat here until the 2026-07-10 ablation on real
+    content measured it at +0.02 recall / −0.005 MRR — no earned keep for its
+    ~90MB model and per-query cost — so it was removed; the fused RRF order is
+    the final order.)
 
     `kind` restricts the search to one content space — the agent's planner
     sets it when it decides the question needs the reference catalog rather
-    than tutorial prose. `rerank_fn` (tests) replaces the real reranker and
-    forces the rerank path on.
+    than tutorial prose.
     """
     from contextlib import nullcontext
 
@@ -236,21 +236,11 @@ def retrieve(
             for ck in ranked[:3]:
                 print(f"[debug]   {pointers[ck]['heading_path']} | {pointers[ck]['url']}")
 
-    from index import rerank as rerank_mod
-
-    # rerank reads a slate wider than k; round-robin order is prefix-stable, so
-    # with reranking off (or failing open) the first k match today's behavior
-    use_rerank = rerank_fn is not None or rerank_mod.enabled()
-    slate = max(k, rerank_mod.RERANK_SLATE) if use_rerank else k
-    ranked = _interleave_pools(pools, dists, slate)
-
-    if use_rerank and len(ranked) > 1:
-        rerank_fn = rerank_fn or rerank_mod.rerank
-        ranked = [p["chunk_key"] for p in rerank_fn(query, [pointers[ck] for ck in ranked], k=k)]
+    ranked = _interleave_pools(pools, dists, k)
 
     # exact API lookup: if the user typed a precise symbol and its reference
     # page was found, pin it first — the docs-search behavior users expect
-    # (an explicit identifier outranks even the cross-encoder's judgment)
+    # (an explicit identifier outranks the fused ranking)
     if symbol:
         exact = next(
             (ck for kd, pl in pools for ck in pl if is_exact_api(pointers[ck], symbol)),

tests/conftest.py

@@ -5,21 +5,13 @@ and agent/translate.py caches default-path translations. Both are deliberate
 in production — state must be shared across requests — but they would leak
 between tests: a model named "model-a" cooled down by one test would be
 silently skipped in the next.
-
-The cross-encoder reranker (on by default in production) is switched off for
-the unit suite — no test may download a 90MB model. Tests that exercise the
-rerank path inject a fake scorer via retrieve(rerank_fn=...) instead.
 """
 
-import os
-
 import pytest
 
 from agent.llm import reset_cooldowns
 from agent.translate import _translate_default
 
-os.environ.setdefault("TORCHDOCS_RERANK", "0")
-
 
 @pytest.fixture(autouse=True)
 def _reset_shared_llm_state():

tests/index/test_rerank.py

@@ -1,106 +0,0 @@
-"""The rerank stage — scoring text, ordering, fail-open, and retrieve() wiring."""
-
-from index.rerank import enabled, rerank, rerank_text
-from index.retrieve import retrieve
-
-
-def _ptr(key, url="https://docs.pytorch.org/docs/stable/generated/torch.nn.Linear.html",
-         title="Linear", heading="Linear", kind="api"):
-    return {
-        "chunk_key": key, "url": url, "anchor": "", "page_title": title,
-        "heading_path": heading, "library": "core", "kind": kind,
-        "source_link": "", "part": 0,
-    }
-
-
-def test_enabled_reads_env_per_call(monkeypatch):
-    monkeypatch.setenv("TORCHDOCS_RERANK", "0")
-    assert not enabled()
-    monkeypatch.setenv("TORCHDOCS_RERANK", "1")
-    assert enabled()
-
-
-def test_rerank_text_includes_symbol_title_heading_and_gloss(monkeypatch):
-    url = "https://docs.pytorch.org/docs/stable/generated/torch.nn.Linear.html"
-    glosses = {url: "The standard fully-connected layer applying a weight matrix and bias."}
-    text = rerank_text(_ptr("a", url=url), glosses)
-    assert "torch.nn.Linear" in text  # symbol from the url
-    assert "Linear" in text
-    assert "fully-connected layer" in text  # the gloss is the semantic payload
-
-
-def test_rerank_orders_by_score_and_cuts_to_k(monkeypatch):
-    monkeypatch.setattr("index.embed.load_glosses", lambda: {})
-    ptrs = [_ptr("low"), _ptr("high"), _ptr("mid")]
-    scores = {"low": 0.1, "high": 0.9, "mid": 0.5}
-
-    def scorer(pairs):
-        # pairs align with ptrs order; look scores up by position
-        return [scores[p["chunk_key"]] for p in ptrs]
-
-    out = rerank("q", ptrs, k=2, scorer=scorer)
-    assert [p["chunk_key"] for p in out] == ["high", "mid"]
-
-
-def test_rerank_fails_open_to_the_incoming_order(monkeypatch):
-    monkeypatch.setattr("index.embed.load_glosses", lambda: {})
-
-    def broken(pairs):
-        raise RuntimeError("model exploded")
-
-    ptrs = [_ptr("first"), _ptr("second"), _ptr("third")]
-    out = rerank("q", ptrs, k=2, scorer=broken)
-    assert [p["chunk_key"] for p in out] == ["first", "second"]
-
-
-def test_rerank_single_candidate_skips_scoring():
-    # no glosses/model needed — must not even build pairs
-    out = rerank("q", [_ptr("only")], k=8, scorer=None)
-    assert [p["chunk_key"] for p in out] == ["only"]
-
-
-# --- retrieve() wiring -------------------------------------------------------
-
-from tests.index.test_retrieve import FakeConn, _row  # noqa: E402 — shared fakes
-
-
-def test_retrieve_feeds_a_wide_slate_to_the_reranker_and_keeps_its_order():
-    refs = [_row(f"ref{i}", kind="api", dist=0.2 + i / 100) for i in range(10)]
-    conn = FakeConn([refs, [], [], [], [], []])
-    seen = {}
-
-    def fake_rerank(query, pointers, k):
-        seen["slate"] = len(pointers)
-        return list(reversed(pointers))[:k]  # deliberately invert the fused order
-
-    results = retrieve(
-        "descriptive question", k=3, conn=conn,
-        embed_fn=lambda q: [0.0] * 384, rerank_fn=fake_rerank,
-    )
-    assert seen["slate"] == 10  # the whole fused slate, not just top-k
-    assert [r["chunk_key"] for r in results] == ["ref9", "ref8", "ref7"]
-
-
-def test_retrieve_skips_reranker_when_disabled(monkeypatch):
-    monkeypatch.setenv("TORCHDOCS_RERANK", "0")
-    refs = [_row(f"ref{i}", kind="api", dist=0.2) for i in range(4)]
-    conn = FakeConn([refs, [], [], [], [], []])
-    results = retrieve("q", k=2, conn=conn, embed_fn=lambda q: [0.0] * 384)
-    # fused order preserved — no reranker touched it (none injected, env off)
-    assert [r["chunk_key"] for r in results] == ["ref0", "ref1"]
-
-
-def test_exact_symbol_pin_outranks_the_reranker():
-    exact = _row("exact", kind="api", url="https://docs.pytorch.org/docs/stable/generated/torch.optim.SGD.html")
-    refs = [_row(f"ref{i}", kind="api", dist=0.2) for i in range(3)]
-    # query pools: dense, keyword, then the symbol channel returns the exact page
-    conn = FakeConn([refs, [], [], [], [], [], [exact]])
-
-    def fake_rerank(query, pointers, k):
-        return [p for p in pointers if p["chunk_key"] != "exact"][:k]  # reranker demotes it
-
-    results = retrieve(
-        "how do I use torch.optim.SGD?", k=3, conn=conn,
-        embed_fn=lambda q: [0.0] * 384, rerank_fn=fake_rerank,
-    )
-    assert results[0]["chunk_key"] == "exact"  # the pin wins anyway