diff --git a/shelfmark/release_sources/direct_download/annas_archive.py b/shelfmark/release_sources/direct_download/annas_archive.py index 6978663..4bd1b65 100644 --- a/shelfmark/release_sources/direct_download/annas_archive.py +++ b/shelfmark/release_sources/direct_download/annas_archive.py @@ -1392,10 +1392,15 @@ def _extract_libgen_download_url(link: str, cancel_flag: Event | None = None) -> base_url = "/".join(link.split("/")[:3]) logger.debug("Libgen fast: trying %s", link) + # libgen.li's ads.php returns an empty 200 body to requests without a Referer (an + # anti-hotlinking check the mirrors added). A same-origin Referer is enough to get the + # real page back. + headers = {**downloader.DOWNLOAD_HEADERS, "Referer": f"{base_url}/"} + try: response = requests.get( link, - headers=downloader.DOWNLOAD_HEADERS, + headers=headers, timeout=(5, 10), allow_redirects=True, proxies=network.get_proxies(link), diff --git a/shelfmark/release_sources/libgen/scraper.py b/shelfmark/release_sources/libgen/scraper.py index c3a6f9a..2d6fdc7 100644 --- a/shelfmark/release_sources/libgen/scraper.py +++ b/shelfmark/release_sources/libgen/scraper.py @@ -8,7 +8,7 @@ needed. All shelfmark-stateful behaviour lives in source.py/handler.py. import re from http import HTTPStatus -from urllib.parse import quote +from urllib.parse import quote, urlsplit import requests from bs4 import BeautifulSoup, Tag @@ -79,10 +79,15 @@ def fetch_page(url: str, timeout: tuple[int, int] = (5, 15)) -> str | None: and tests patch it. Uses the app's proxy/SSL/DNS configuration so egress stays on whatever network the container is bound to (the VPN namespace, in the deployed stack). """ + # libgen.li's ads.php returns an empty 200 body to requests without a Referer (an + # anti-hotlinking check the mirrors added). A same-origin Referer is enough and is + # harmless for the search page, so send one for every fetch. + parts = urlsplit(url) + headers = {**downloader.DOWNLOAD_HEADERS, "Referer": f"{parts.scheme}://{parts.netloc}/"} try: response = requests.get( url, - headers=downloader.DOWNLOAD_HEADERS, + headers=headers, timeout=timeout, allow_redirects=True, proxies=network.get_proxies(url), diff --git a/tests/direct_download/test_libgen_extract.py b/tests/direct_download/test_libgen_extract.py new file mode 100644 index 0000000..78aea6b --- /dev/null +++ b/tests/direct_download/test_libgen_extract.py @@ -0,0 +1,33 @@ +"""Tests for the direct-download Libgen ads.php resolution (AA-md5 -> libgen fallback).""" + +from unittest.mock import patch + +from shelfmark.release_sources.direct_download import annas_archive +from tests.libgen import sample_html as html + + +def test_extract_libgen_download_url_sends_same_origin_referer(): + # libgen.li's ads.php returns an empty 200 without a Referer; the resolver must send a + # same-origin one or it never finds the get.php link and the download silently fails. + captured = {} + + class FakeResponse: + status_code = 200 + text = html.ADS_HTML + url = "https://libgen.li/ads.php?md5=" + html.MD5_A + + def fake_get(link, **kwargs): + captured["headers"] = kwargs["headers"] + return FakeResponse() + + with ( + patch.object(annas_archive.requests, "get", side_effect=fake_get), + patch.object(annas_archive.network, "get_proxies", return_value=None), + patch.object(annas_archive.network, "get_ssl_verify", return_value=True), + ): + url = annas_archive._extract_libgen_download_url( + f"https://libgen.li/ads.php?md5={html.MD5_A}" + ) + + assert captured["headers"]["Referer"] == "https://libgen.li/" + assert url == f"https://libgen.li/get.php?md5={html.MD5_A}&key={html.GET_KEY}" diff --git a/tests/libgen/test_scraper.py b/tests/libgen/test_scraper.py index 1f70a8f..8a01ed0 100644 --- a/tests/libgen/test_scraper.py +++ b/tests/libgen/test_scraper.py @@ -72,6 +72,30 @@ def test_resolve_download_url_missing_get_returns_none(): assert scraper.resolve_download_url(html.ADS_HTML_NO_GET, "https://libgen.li") is None +def test_fetch_page_sends_same_origin_referer(): + # libgen.li's ads.php returns an empty 200 without a Referer; fetch_page must send a + # same-origin one or every download-page fetch comes back blank. + captured = {} + + class FakeResponse: + status_code = 200 + text = "ok" + + def fake_get(url, **kwargs): + captured["headers"] = kwargs["headers"] + return FakeResponse() + + with ( + patch.object(scraper.requests, "get", side_effect=fake_get), + patch.object(scraper.network, "get_proxies", return_value=None), + patch.object(scraper.network, "get_ssl_verify", return_value=True), + ): + result = scraper.fetch_page("https://libgen.li/ads.php?md5=abc") + + assert result == "ok" + assert captured["headers"]["Referer"] == "https://libgen.li/" + + def test_search_libgen_falls_through_dead_mirror(): calls = []