fix(sources): send a Referer when fetching libgen ads.php pages (#1340)

## What

libgen.li's `ads.php?md5=` now returns an **empty `200`** to any request
without a `Referer` — an anti-hotlinking check the mirrors added
recently. Both libgen paths fetch it without one, so the page comes back
blank and the download silently fails while **search keeps working**
(which is exactly why it looks like rate-limiting or mirror drift rather
than a bug).

Same one-line cause, two call sites: the Libgen search source
(`libgen/scraper.py:fetch_page`) and the AA-md5 → libgen fallback
(`direct_download/annas_archive.py:_extract_libgen_download_url`). Fix:
send a same-origin `Referer: <scheme>://<host>/` on the `ads.php` fetch
in both.

## Worth a look in review

- **The referer goes on the *resolution* fetch, not the download.**
`download_url(..., referer=...)` was already correct — the blank page
happens one step earlier, at the `ads.php` GET.
- Reproduced against live mirrors: `ads.php` returns `Content-Length: 0`
bare, the full page with a `Referer`, and resolvable files download
valid bytes again.

Regression tests in `tests/libgen/` and `tests/direct_download/` assert
the header on both paths. Lint/format/typecheck clean.

Follow-up to #1326.

shelfmark/release_sources/direct_download/annas_archive.py

@@ -1392,10 +1392,15 @@ def _extract_libgen_download_url(link: str, cancel_flag: Event | None = None) ->
     base_url = "/".join(link.split("/")[:3])
     logger.debug("Libgen fast: trying %s", link)
 
+    # libgen.li's ads.php returns an empty 200 body to requests without a Referer (an
+    # anti-hotlinking check the mirrors added). A same-origin Referer is enough to get the
+    # real page back.
+    headers = {**downloader.DOWNLOAD_HEADERS, "Referer": f"{base_url}/"}
+
     try:
         response = requests.get(
             link,
-            headers=downloader.DOWNLOAD_HEADERS,
+            headers=headers,
             timeout=(5, 10),
             allow_redirects=True,
             proxies=network.get_proxies(link),

shelfmark/release_sources/libgen/scraper.py

@@ -8,7 +8,7 @@ needed. All shelfmark-stateful behaviour lives in source.py/handler.py.
 
 import re
 from http import HTTPStatus
-from urllib.parse import quote
+from urllib.parse import quote, urlsplit
 
 import requests
 from bs4 import BeautifulSoup, Tag
@@ -79,10 +79,15 @@ def fetch_page(url: str, timeout: tuple[int, int] = (5, 15)) -> str | None:
     and tests patch it. Uses the app's proxy/SSL/DNS configuration so egress stays on
     whatever network the container is bound to (the VPN namespace, in the deployed stack).
     """
+    # libgen.li's ads.php returns an empty 200 body to requests without a Referer (an
+    # anti-hotlinking check the mirrors added). A same-origin Referer is enough and is
+    # harmless for the search page, so send one for every fetch.
+    parts = urlsplit(url)
+    headers = {**downloader.DOWNLOAD_HEADERS, "Referer": f"{parts.scheme}://{parts.netloc}/"}
     try:
         response = requests.get(
             url,
-            headers=downloader.DOWNLOAD_HEADERS,
+            headers=headers,
             timeout=timeout,
             allow_redirects=True,
             proxies=network.get_proxies(url),

tests/direct_download/test_libgen_extract.py

@@ -0,0 +1,33 @@
+"""Tests for the direct-download Libgen ads.php resolution (AA-md5 -> libgen fallback)."""
+
+from unittest.mock import patch
+
+from shelfmark.release_sources.direct_download import annas_archive
+from tests.libgen import sample_html as html
+
+
+def test_extract_libgen_download_url_sends_same_origin_referer():
+    # libgen.li's ads.php returns an empty 200 without a Referer; the resolver must send a
+    # same-origin one or it never finds the get.php link and the download silently fails.
+    captured = {}
+
+    class FakeResponse:
+        status_code = 200
+        text = html.ADS_HTML
+        url = "https://libgen.li/ads.php?md5=" + html.MD5_A
+
+    def fake_get(link, **kwargs):
+        captured["headers"] = kwargs["headers"]
+        return FakeResponse()
+
+    with (
+        patch.object(annas_archive.requests, "get", side_effect=fake_get),
+        patch.object(annas_archive.network, "get_proxies", return_value=None),
+        patch.object(annas_archive.network, "get_ssl_verify", return_value=True),
+    ):
+        url = annas_archive._extract_libgen_download_url(
+            f"https://libgen.li/ads.php?md5={html.MD5_A}"
+        )
+
+    assert captured["headers"]["Referer"] == "https://libgen.li/"
+    assert url == f"https://libgen.li/get.php?md5={html.MD5_A}&key={html.GET_KEY}"

tests/libgen/test_scraper.py

@@ -72,6 +72,30 @@ def test_resolve_download_url_missing_get_returns_none():
     assert scraper.resolve_download_url(html.ADS_HTML_NO_GET, "https://libgen.li") is None
 
 
+def test_fetch_page_sends_same_origin_referer():
+    # libgen.li's ads.php returns an empty 200 without a Referer; fetch_page must send a
+    # same-origin one or every download-page fetch comes back blank.
+    captured = {}
+
+    class FakeResponse:
+        status_code = 200
+        text = "<html>ok</html>"
+
+    def fake_get(url, **kwargs):
+        captured["headers"] = kwargs["headers"]
+        return FakeResponse()
+
+    with (
+        patch.object(scraper.requests, "get", side_effect=fake_get),
+        patch.object(scraper.network, "get_proxies", return_value=None),
+        patch.object(scraper.network, "get_ssl_verify", return_value=True),
+    ):
+        result = scraper.fetch_page("https://libgen.li/ads.php?md5=abc")
+
+    assert result == "<html>ok</html>"
+    assert captured["headers"]["Referer"] == "https://libgen.li/"
+
+
 def test_search_libgen_falls_through_dead_mirror():
     calls = []