Follow client-side redirects + read_page URL guard: real API content (#92)

* Follow client-side redirects: the core torch API reference was empty stubs

Measured from the committed index manifest: 3,435 of 4,517 crawled pages had
title "Redirecting…" with a single empty chunk — every one a
docs.pytorch.org/docs/stable/... API reference page (torch.optim.SGD,
torch.nn.Linear, CrossEntropyLoss — exactly the pages retrieval was tuned on).
docs/stable/ serves a client-side redirect stub whose only real content is a
`<meta http-equiv="refresh" content="0; url=<versioned>">`; requests follows
HTTP 3xx but not this, so the crawler snapshotted empty stubs and answers on
core API pages came back hollow ("the context only contains redirect notices").
Retrieval still scored well because it matches symbol + gloss + synopsis, not
the (empty) body — so the ceiling was hidden until an answer exposed it.

ingest/discover.py: redirect_target() reads the meta-refresh (or a differing
<link rel=canonical>) target, and fetch_html() follows it, hop-bounded and
loop-protected, returning the real page's HTML while the caller keeps its
ORIGINAL url as the pointer/citation key (stable URLs stay stable in the index
and in answers). Both the crawl (ingest/crawl.py) and live hydration
(index/hydrate.py) now go through fetch_html, so a re-crawl fills real content
and the deployed Space stops serving redirect stubs.

Also: questions pass --sleep 0 (the 429-retry reads the server's own
Retry-After — the real rate limiter — instead of a guessed proactive delay).

220 tests pass (redirect parsing: meta-refresh, canonical fallback, real-page
negative, follow-through, loop protection).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LX1gm8fuzK2ZeEWi3Zar1b

* read_page: reject a section heading passed as a URL instead of fetching it

Live Space logs showed the planner calling read_page with a section heading it
saw in a search result — "Build the Neural Network > Define the Class" — as the
url. hydrate_page → fetch then failed with "No scheme supplied", wasting a tool
call and printing a scary error. read_page now rejects a non-http url up front
and returns a corrective error ("needs the full https:// URL from a search
result's url field"), so the planner self-corrects instead of burning a step.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LX1gm8fuzK2ZeEWi3Zar1b

---------

Co-authored-by: Claude <noreply@anthropic.com>

.github/workflows/build-index.yml

@@ -91,7 +91,7 @@ jobs:
           OPENAI_COMPAT_BASE_URL: https://openrouter.ai/api/v1
           OPENAI_COMPAT_API_KEY: ${{ secrets.OPENROUTER }}
           TORCHDOCS_OPENAI_COMPAT_MODEL: ${{ inputs.llm || 'tencent/hy3:free,meta-llama/llama-3.3-70b-instruct:free' }}
-        run: python scripts/generate_questions.py --limit 0 --batch 25 --sleep 3
+        run: python scripts/generate_questions.py --limit 0 --batch 25 --sleep 0
       - name: Commit any new enrichment (also on failure — partial progress is saved)
         if: always()
         run: |

.github/workflows/generate-glosses.yml

@@ -78,7 +78,7 @@ jobs:
         # 5 questions/page → batch 25 = ~125 short questions/reply, ~145 calls
         # for the whole corpus (vs ~363 at batch 10). hy3 handles the longer
         # reply fine; fewer calls means far less time in the free-tier queue.
-        run: python scripts/generate_questions.py --limit "${{ inputs.limit }}" --batch 25 --sleep 3
+        run: python scripts/generate_questions.py --limit "${{ inputs.limit }}" --batch 25 --sleep 0
       - name: Commit whatever was written (also on failure — the run is resumable)
         if: always()
         run: |

agent/tools.py

@@ -80,6 +80,17 @@ def read_page(url: str) -> dict:
     """Whole page for a URL already surfaced by search_docs."""
     from index.hydrate import hydrate_page
 
+    url = (url or "").strip()
+    if not url.startswith(("http://", "https://")):
+        # the planner sometimes passes a section HEADING it saw in a search
+        # result (e.g. "Build the Neural Network > Define the Class") instead of
+        # the url. Don't fetch that (it 'No scheme supplied'-errors and wastes a
+        # call) — tell the model exactly what read_page needs so it self-corrects.
+        return {
+            "url": url,
+            "error": "read_page needs the full https:// URL from a search_docs "
+            "result's `url` field, not a section title.",
+        }
     page = hydrate_page(url)
     if page is None:
         return {"url": url, "error": "page not in the snapshot"}

index/hydrate.py

@@ -29,10 +29,12 @@ def _live_page(url: str) -> tuple[dict, str] | None:
     if not _LIVE:
         return None
     from ingest.crawl import extract_main_html, to_markdown
-    from ingest.discover import fetch
+    from ingest.discover import fetch_html
 
     try:
-        html = fetch(url).decode("utf-8", errors="replace")
+        # follow client-side redirects (docs/stable/... → versioned page); a
+        # plain fetch would return the "Redirecting…" stub and hollow the answer
+        html = fetch_html(url)
     except Exception as exc:  # noqa: BLE001 — a dead link just means "no content"
         print(f"[hydrate] live fetch failed for {url}: {exc}", flush=True)
         return None

ingest/crawl.py

@@ -102,7 +102,7 @@ def crawl(
     import os
     import time
 
-    from ingest.discover import fetch
+    from ingest.discover import fetch_html
 
     delay = float(os.environ.get("TORCHDOCS_CRAWL_DELAY", "0.2"))
     changed: dict[str, int] = {}
@@ -110,7 +110,9 @@ def crawl(
         count = 0
         for url in sorted(urls):
             try:
-                html = fetch(url).decode("utf-8", errors="replace")
+                # follow client-side redirects: docs/stable/... serves a
+                # "Redirecting…" stub whose real content is behind a meta-refresh
+                html = fetch_html(url)
             except Exception as exc:  # noqa: BLE001 — one bad page must not kill the crawl
                 print(f"[crawl] {url}: fetch failed ({exc})")
                 continue

ingest/discover.py

@@ -132,6 +132,57 @@ def fetch(url: str, timeout: float = 30.0, retries: int = FETCH_RETRIES) -> byte
     raise last_exc  # type: ignore[misc]  # loop ran ≥1 time, so last_exc is set
 
 
+def redirect_target(html: str, base_url: str) -> str | None:
+    """The URL a client-side redirect stub points at, else None.
+
+    docs.pytorch.org/docs/stable/<...> serves a "Redirecting…" page whose only
+    real content is `<meta http-equiv="refresh" content="0; url=<versioned>">`.
+    requests follows HTTP 3xx but NOT this, so a plain fetch captured an empty
+    stub for the entire core API reference (measured: 3,435/4,517 crawled pages
+    had title "Redirecting…", every one a docs/stable API page). A real content
+    page never carries a refresh meta, so finding one unambiguously means "this
+    is a redirect — follow it". Falls back to <link rel="canonical"> when it
+    points elsewhere.
+    """
+    from urllib.parse import urljoin
+
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+    meta = soup.find("meta", attrs={"http-equiv": re.compile(r"^refresh$", re.I)})
+    if meta and meta.get("content"):
+        m = re.search(r"url\s*=\s*(.+)$", meta["content"], re.I)
+        if m:
+            return urljoin(base_url, m.group(1).strip().strip("'\""))
+    canonical = soup.find("link", rel="canonical")
+    if canonical and canonical.get("href"):
+        target = urljoin(base_url, canonical["href"].strip())
+        if target.rstrip("/") != base_url.rstrip("/"):
+            return target
+    return None
+
+
+def fetch_html(url: str, *, max_hops: int = 3, **kwargs) -> str:
+    """fetch() + follow client-side (meta-refresh) redirects to the real page.
+
+    Returns the final page's HTML as text. Loop-protected and hop-bounded so a
+    misconfigured redirect chain can't spin forever. The caller keeps its
+    ORIGINAL url as the pointer/citation key — only the *content* comes from the
+    redirect target — so stable URLs stay stable in the index and in answers.
+    """
+    seen: set[str] = set()
+    html = ""
+    for _ in range(max_hops + 1):
+        html = fetch(url, **kwargs).decode("utf-8", errors="replace")
+        seen.add(url)
+        target = redirect_target(html, url)
+        if not target or target in seen:
+            return html
+        print(f"[fetch] following redirect {url} → {target}", flush=True)
+        url = target
+    return html  # hop budget exhausted → return the last page fetched
+
+
 def _sitemap_pages(base: str, xml_text: str) -> set[str]:
     """Page URLs from a sitemap, following one level of <sitemapindex>."""
     import requests

tests/agent/test_tools.py

@@ -45,6 +45,18 @@ def test_read_page_missing(monkeypatch):
     assert "error" in read_page("https://x")
 
 
+def test_read_page_rejects_a_heading_instead_of_a_url(monkeypatch):
+    # the planner sometimes passes a section heading it saw in a search result;
+    # read_page must not try to fetch it (No scheme supplied) — it returns a
+    # corrective error WITHOUT touching hydrate_page
+    def must_not_fetch(url):  # pragma: no cover
+        raise AssertionError("hydrate_page must not run on a non-URL")
+
+    monkeypatch.setattr("index.hydrate.hydrate_page", must_not_fetch)
+    out = read_page("Build the Neural Network > Define the Class")
+    assert "error" in out and "URL" in out["error"]
+
+
 def test_search_docs_passes_kind_to_retrieve(monkeypatch):
     import agent.tools as tools
 

tests/ingest/test_discover.py

@@ -171,3 +171,69 @@ def test_fetch_abandons_oversized_pages_early(monkeypatch):
     monkeypatch.setattr("requests.get", lambda url, **kw: huge)
     with pytest.raises(ValueError, match="exceeds"):
         disc.fetch("https://docs.pytorch.org/huge.html")
+
+
+# --- client-side redirect following (docs/stable "Redirecting…" stubs) --------
+
+
+def test_redirect_target_reads_meta_refresh():
+    from ingest.discover import redirect_target
+
+    stub = (
+        '<html><head><title>Redirecting…</title>'
+        '<meta http-equiv="refresh" content="0; url=../../2.13/generated/torch.optim.SGD.html">'
+        "</head><body>You should have been redirected.</body></html>"
+    )
+    base = "https://docs.pytorch.org/docs/stable/generated/torch.optim.SGD.html"
+    assert (
+        redirect_target(stub, base)
+        == "https://docs.pytorch.org/docs/2.13/generated/torch.optim.SGD.html"
+    )
+
+
+def test_redirect_target_falls_back_to_canonical():
+    stub = (
+        '<html><head><link rel="canonical" '
+        'href="https://docs.pytorch.org/docs/2.13/x.html"></head><body/></html>'
+    )
+    from ingest.discover import redirect_target
+
+    assert redirect_target(stub, "https://docs.pytorch.org/docs/stable/x.html") == (
+        "https://docs.pytorch.org/docs/2.13/x.html"
+    )
+
+
+def test_redirect_target_none_for_a_real_page():
+    from ingest.discover import redirect_target
+
+    real = "<html><head><title>torch.optim.SGD</title></head><body><h1>SGD</h1></body></html>"
+    # a canonical that points at the SAME page is not a redirect
+    same = (
+        '<html><head><link rel="canonical" href="https://d/p.html"></head><body>real</body></html>'
+    )
+    assert redirect_target(real, "https://d/p.html") is None
+    assert redirect_target(same, "https://d/p.html") is None
+
+
+def test_fetch_html_follows_the_stub_to_the_real_content(monkeypatch):
+    import ingest.discover as disc
+
+    real = "<html><body><h1>SGD</h1><p>momentum...</p></body></html>"
+    stub = (
+        '<meta http-equiv="refresh" content="0; url=https://d/real.html">'
+    )
+    pages = {"https://d/stable.html": stub, "https://d/real.html": real}
+    monkeypatch.setattr(disc, "fetch", lambda url, **kw: pages[url].encode())
+    assert disc.fetch_html("https://d/stable.html") == real
+
+
+def test_fetch_html_is_loop_protected(monkeypatch):
+    import ingest.discover as disc
+
+    # two stubs pointing at each other must not spin forever
+    a = '<meta http-equiv="refresh" content="0; url=https://d/b.html">'
+    b = '<meta http-equiv="refresh" content="0; url=https://d/a.html">'
+    pages = {"https://d/a.html": a, "https://d/b.html": b}
+    monkeypatch.setattr(disc, "fetch", lambda url, **kw: pages[url].encode())
+    # terminates (returns the last fetched html) instead of hanging
+    assert disc.fetch_html("https://d/a.html", max_hops=3) in (a, b)