diff --git a/shelfmark/core/image_cache.py b/shelfmark/core/image_cache.py index d28ac404..21ea7317 100644 --- a/shelfmark/core/image_cache.py +++ b/shelfmark/core/image_cache.py @@ -3,7 +3,6 @@ import hashlib import ipaddress import json -import socket import threading import time from http import HTTPStatus @@ -15,7 +14,7 @@ import requests from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import coerce_int -from shelfmark.download.network import get_ssl_verify +from shelfmark.download.network import get_ssl_verify, hostname_resolves_publicly if TYPE_CHECKING: from pathlib import Path @@ -526,13 +525,7 @@ class ImageCacheService: if not hostname: return None - try: - resolved = socket.getaddrinfo(hostname, None) - for _, _, _, _, sockaddr in resolved: - ip = ipaddress.ip_address(sockaddr[0]) - if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: - return None - except socket.gaierror, ValueError: + if not hostname_resolves_publicly(hostname, use_custom_dns=True): return None return prepared_url diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index 3df87e23..585428cd 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -518,6 +518,11 @@ def html_get_page( ) original_url = url current_url = selector.rewrite(original_url) + try: + network.ensure_public_destination(current_url) + except network.UnsafeDestinationError as e: + logger.warning("Blocked page fetch: %s", e) + return _fail("The page address is not a public host.", current_url) use_bypasser_now = use_bypasser # Survives across attempts so a cookie won once is still presented on later retries. handshake_cookies: dict[str, str] = {} @@ -572,6 +577,7 @@ def html_get_page( headers=headers, allow_redirects=allow_redirects, verify=get_ssl_verify(current_url), + hooks=network.REDIRECT_GUARD_HOOKS, ) # Z-Library gates the first hit with a 503 that carries nothing but a @@ -879,6 +885,12 @@ def download_url( """Download content from URL with automatic retry and resume support.""" selector = _selector or network.AAMirrorSelector() current_url = selector.rewrite(link) + try: + network.ensure_public_destination(current_url) + except network.UnsafeDestinationError as e: + logger.warning("Blocked download: %s", e) + return None + network.ensure_public_destination(current_url) # Build headers with optional referer headers = DOWNLOAD_HEADERS.copy() @@ -914,6 +926,7 @@ def download_url( cookies=cookies, headers=headers, verify=get_ssl_verify(current_url), + hooks=network.REDIRECT_GUARD_HOOKS, ) response.raise_for_status() @@ -1077,6 +1090,7 @@ def _try_resume( headers=resume_headers, cookies=cookies, verify=get_ssl_verify(url), + hooks=network.REDIRECT_GUARD_HOOKS, ) # Check resume support diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index b47f5812..298b13e0 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -549,6 +549,91 @@ def _is_local_address(host_str: str) -> bool: return addr.is_private or addr.is_loopback or addr.is_link_local +class UnsafeDestinationError(requests.exceptions.InvalidURL): + """A request target resolves to a loopback, private or otherwise non-public address.""" + + +def _address_is_public(address: str) -> bool: + ip = ipaddress.ip_address(address.split("%", 1)[0]) + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None: + ip = ip.ipv4_mapped + return ip.is_global + + +def _system_getaddrinfo(hostname: str, port: None) -> list[Any]: + return original_getaddrinfo(hostname, port) + + +def hostname_resolves_publicly( + hostname: str, *, allow_unresolved: bool = False, use_custom_dns: bool = False +) -> bool: + """Return True when every address ``hostname`` resolves to is globally routable. + + A name that does not resolve counts as public only with ``allow_unresolved``, so the + caller's own request fails on it the way it always did. The check asks the system + resolver unless ``use_custom_dns`` is set: a failed lookup through the patched + resolver switches the DNS provider, and a pre-flight check must not do that. + """ + resolver = socket.getaddrinfo if use_custom_dns else _system_getaddrinfo + try: + return _address_is_public(hostname) + except ValueError: + pass # not an IP literal, resolve it + try: + resolved = resolver(hostname, None) + return all(_address_is_public(str(sockaddr[0])) for *_, sockaddr in resolved) + except socket.gaierror: + return allow_unresolved + except ValueError: + return False + + +def _configured_mirror_hosts() -> set[str]: + """Hostnames of every mirror the operator configured, which are trusted as given.""" + from shelfmark.core import mirrors + + urls = [ + *mirrors.get_aa_mirrors(), + *mirrors.get_libgen_mirrors(), + *mirrors.get_zlib_mirrors(), + *mirrors.get_welib_mirrors(), + *mirrors.get_oceanofpdf_mirrors(), + ] + return {(urllib.parse.urlparse(url).hostname or "").lower() for url in urls} + + +def ensure_public_destination(url: str) -> None: + """Refuse a URL whose host resolves to a non-public address. + + Direct-download pages and file links come from remote sites or from API callers, so a + hostile value could otherwise aim the server at loopback, the LAN or a sibling + container. A host the operator configured as a mirror is allowed as written, and so is + a name that does not resolve (the request fails on it normally). + """ + parsed = urllib.parse.urlparse(url) + hostname = (parsed.hostname or "").lower().rstrip(".") + if parsed.scheme not in ("http", "https") or not hostname: + msg = f"Refusing to request {url!r}: not an http(s) URL with a host" + raise UnsafeDestinationError(msg) + if hostname in _configured_mirror_hosts(): + return + if not hostname_resolves_publicly(hostname, allow_unresolved=True): + msg = f"Refusing to request {hostname!r}: it resolves to a non-public address" + raise UnsafeDestinationError(msg) + + +def _guard_redirect_hop(response: requests.Response, *_args: object, **_kwargs: object) -> None: + """Response hook: check where a redirect points before requests follows it.""" + if response.is_redirect: + location = response.headers.get("Location", "") + if location: + ensure_public_destination(urllib.parse.urljoin(response.url, location)) + + +# Pass as ``hooks=`` so every redirect hop of a request is checked, not just its first URL. +REDIRECT_GUARD_HOOKS: dict[str, Any] = {"response": [_guard_redirect_hop]} + + def _is_ip_address(host_str: str) -> bool: """Check if a string is a valid IP address (IPv4 or IPv6).""" try: diff --git a/shelfmark/release_sources/direct_download/oceanofpdf.py b/shelfmark/release_sources/direct_download/oceanofpdf.py index 51cb99a5..53fc997b 100644 --- a/shelfmark/release_sources/direct_download/oceanofpdf.py +++ b/shelfmark/release_sources/direct_download/oceanofpdf.py @@ -67,6 +67,9 @@ _HTML_SIGNATURES = (b" bool: return False +def _is_site_url(url: str) -> bool: + """Whether ``url`` is a page of the configured OceanofPDF mirror itself. + + The book page, the download form's action and any redirect between them must stay on + the mirror, same scheme and port, so a record or a page cannot point the fetch at + another host. The signed file link is the one URL that legitimately sits elsewhere. + """ + base_url = _base_url() + if not base_url: + return False + try: + parsed, base = urlparse(url), urlparse(base_url) + return ( + parsed.scheme == base.scheme + and (parsed.hostname or "").lower().rstrip(".") == (base.hostname or "").lower() + and parsed.port == base.port + ) + except ValueError: + return False + + def _search_url(base_url: str, query: str) -> str: return f"{base_url}/?s={quote_plus(query)}" @@ -233,15 +257,34 @@ def _post_fetch_form(action_url: str, fields: dict[str, str], referer: str) -> r "Referer": referer, "Origin": f"{urlparse(referer).scheme}://{urlparse(referer).netloc}", } - return requests.post( - action_url, - data=fields, - headers=headers, - cookies=downloader.get_cf_cookies_for_domain(hostname), - timeout=_POST_TIMEOUT, - proxies=network.get_proxies(action_url), - verify=network.get_ssl_verify(action_url), - ) + method = "POST" + url = action_url + for _ in range(_MAX_POST_REDIRECTS + 1): + if not _is_site_url(url): + msg = f"OceanofPDF download form points off the mirror: {url}" + raise requests.exceptions.InvalidURL(msg) + network.ensure_public_destination(url) + common = { + "headers": headers, + "cookies": downloader.get_cf_cookies_for_domain(hostname), + "timeout": _POST_TIMEOUT, + "proxies": network.get_proxies(url), + "verify": network.get_ssl_verify(url), + "allow_redirects": False, + } + response = ( + requests.post(url, data=fields, **common) + if method == "POST" + else requests.get(url, **common) + ) + location = response.headers.get("Location") if response.is_redirect else None + if not location: + return response + url = urljoin(url, location) + if response.status_code not in (_HTTP_TEMP_REDIRECT, _HTTP_PERM_REDIRECT): + method = "GET" + msg = "OceanofPDF download form redirected too many times" + raise requests.exceptions.TooManyRedirects(msg) def _log_unresolved_fetch_page(response: requests.Response) -> None: @@ -404,6 +447,12 @@ class OceanofPDFProvider: if not book_url: msg = f"OceanofPDF record {book_info.id!r} has no book page URL" raise RuntimeError(msg) + if not self.is_enabled(): + msg = "OceanofPDF is not enabled" + raise RuntimeError(msg) + if not _is_site_url(book_url): + msg = f"OceanofPDF record {book_info.id!r} does not point at the configured mirror" + raise RuntimeError(msg) def status(message: str) -> None: if status_callback: @@ -432,6 +481,9 @@ class OceanofPDFProvider: return None action, fields = form action_url = urljoin(book_url, action) + if not _is_site_url(action_url): + logger.warning("OceanofPDF download form points off the mirror: %s", action_url) + return None if cancel_flag and cancel_flag.is_set(): return None @@ -441,6 +493,9 @@ class OceanofPDFProvider: logger.warning("OceanofPDF did not return a file link for %s", book_url) return None file_url, delay = resolved + if urlparse(file_url).scheme not in ("http", "https"): + logger.warning("OceanofPDF file link is not an http(s) URL: %s", file_url) + return None wait = min(delay, _MAX_REDIRECT_DELAY_SECONDS) if wait > 0: diff --git a/tests/conftest.py b/tests/conftest.py index fee6198f..de13e702 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,24 @@ def _clear_torrent_fetch_cache(): clear_torrent_fetch_cache() +@pytest.fixture(autouse=True) +def _public_destination_dns(monkeypatch): + """Resolve every host to a public address for the destination guard. + + The guard looks hosts up before a request. Tests use made-up and real hostnames and + must not depend on the DNS of the machine running them (a filtering resolver answers + ``::`` for some mirrors), so the guard's resolver is stubbed. Tests of the guard + itself replace it again. + """ + from shelfmark.download import network + + monkeypatch.setattr( + network, + "_system_getaddrinfo", + lambda *_args, **_kwargs: [(2, 1, 6, "", ("93.184.216.34", 0))], + ) + + @pytest.fixture def sample_prowlarr_result(): """Sample Prowlarr API search result.""" diff --git a/tests/core/test_image_cache.py b/tests/core/test_image_cache.py index 0225ae6f..e0a3fd98 100644 --- a/tests/core/test_image_cache.py +++ b/tests/core/test_image_cache.py @@ -54,7 +54,7 @@ def test_fetch_and_cache_blocks_unsafe_redirect(tmp_path, monkeypatch) -> None: calls.append(url) return RedirectResponse() - monkeypatch.setattr("shelfmark.core.image_cache.socket.getaddrinfo", fake_getaddrinfo) + monkeypatch.setattr("shelfmark.download.network.socket.getaddrinfo", fake_getaddrinfo) monkeypatch.setattr("shelfmark.core.image_cache.requests.get", fake_get) assert cache.fetch_and_cache("cover-redirect", "https://example.com/cover.jpg") is None diff --git a/tests/direct_download/test_oceanofpdf.py b/tests/direct_download/test_oceanofpdf.py index 88fa5c17..831fca42 100644 --- a/tests/direct_download/test_oceanofpdf.py +++ b/tests/direct_download/test_oceanofpdf.py @@ -336,6 +336,8 @@ class _FakeResponse: self.text = text self.status_code = status_code self.url = url + self.headers: dict[str, str] = {} + self.is_redirect = False def raise_for_status(self): if self.status_code >= 400: @@ -577,3 +579,99 @@ def test_mirror_list_is_normalised_and_first_entry_wins(monkeypatch): assert mirrors.get_oceanofpdf_mirrors() == ["https://ocean.example", "https://backup.example"] assert mirrors.get_oceanofpdf_base_url() == "https://ocean.example" + + +# --- Hostile records and pages --- + + +def _hostile_record(url: str) -> BrowseRecord: + return BrowseRecord( + id="oceanofpdf:x", + title="Dune", + source="direct_download", + format="epub", + source_url=url, + ) + + +@pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1:8080/admin", + "http://169.254.169.254/latest/meta-data/", + "https://other.example/book/", + "http://ocean.example/authors/dune/", + "https://ocean.example:8443/authors/dune/", + "file:///etc/passwd", + ], +) +def test_download_never_fetches_a_source_url_off_the_mirror(configured, monkeypatch, tmp_path, url): + requested = _serve_pages(monkeypatch, {url: BOOK_PAGE}) + posts, downloads = _wire_download(monkeypatch, post_responses=[], payload=b"PK" * 10_000) + + with pytest.raises(RuntimeError): + op.OceanofPDFProvider().download( + _hostile_record(url), tmp_path / "x.epub", None, Event(), None + ) + + assert url not in requested + assert posts == [] + assert downloads == [] + + +def test_download_refuses_when_the_provider_is_switched_off(configured, monkeypatch, tmp_path): + configured["SOURCE_PRIORITY"] = [{"id": "oceanofpdf", "enabled": False}] + requested = _serve_pages(monkeypatch, {DUNE_URL: BOOK_PAGE}) + + with pytest.raises(RuntimeError): + op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None) + + assert requested == [] + + +@pytest.mark.parametrize( + "action", + ["http://127.0.0.1:9000/Fetching_Resource.php", "https://evil.example/Fetching_Resource.php"], +) +def test_download_never_posts_to_a_form_action_off_the_mirror( + configured, monkeypatch, tmp_path, action +): + page = BOOK_PAGE.replace("https://ocean.example/Fetching_Resource.php", action) + posts, downloads = _wire_download(monkeypatch, post_responses=[], payload=b"PK" * 10_000) + _serve_pages(monkeypatch, {DUNE_URL: page}) + + result = op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None) + + assert result is None + assert posts == [] + assert downloads == [] + + +def test_download_never_fetches_a_file_link_that_is_not_http(configured, monkeypatch, tmp_path): + interstitial = _page('') + _posts, downloads = _wire_download( + monkeypatch, post_responses=[_FakeResponse(interstitial)], payload=b"PK" * 10_000 + ) + + result = op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None) + + assert result is None + assert downloads == [] + + +def test_form_post_redirect_off_the_mirror_is_not_followed(configured, monkeypatch, tmp_path): + redirect = _FakeResponse("", status_code=302) + redirect.is_redirect = True + redirect.headers = {"Location": "http://127.0.0.1:9000/internal"} + posts, downloads = _wire_download( + monkeypatch, post_responses=[redirect], payload=b"PK" * 10_000 + ) + gets: list[str] = [] + monkeypatch.setattr(op.requests, "get", lambda url, **_kw: gets.append(url)) + + result = op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None) + + assert result is None + assert len(posts) == 1 + assert gets == [] + assert downloads == [] diff --git a/tests/download/test_network_destination_guard.py b/tests/download/test_network_destination_guard.py new file mode 100644 index 00000000..27dfd161 --- /dev/null +++ b/tests/download/test_network_destination_guard.py @@ -0,0 +1,103 @@ +"""The destination guard behind the direct-download fetchers.""" + +from types import SimpleNamespace + +import pytest + +from shelfmark.core import mirrors +from shelfmark.download import network + + +def _resolve_to(monkeypatch, table: dict[str, str]) -> None: + def fake_getaddrinfo(host, *_args, **_kwargs): + if host not in table: + raise network.socket.gaierror(-2, "Name or service not known") + return [(None, None, None, None, (table[host], 0))] + + monkeypatch.setattr(network, "_system_getaddrinfo", fake_getaddrinfo) + + +@pytest.fixture(autouse=True) +def no_mirrors(monkeypatch): + monkeypatch.setattr(mirrors, "_get_config", lambda: {}) + + +@pytest.mark.parametrize( + "address", + [ + "127.0.0.1", + "10.1.2.3", + "172.17.0.2", + "192.168.1.5", + "169.254.169.254", + "100.64.0.1", + "0.0.0.0", + "::1", + "fe80::1", + "fd00::1", + "::ffff:127.0.0.1", + "::ffff:10.0.0.1", + ], +) +def test_non_public_addresses_are_refused(monkeypatch, address): + _resolve_to(monkeypatch, {"evil.example": address}) + + with pytest.raises(network.UnsafeDestinationError): + network.ensure_public_destination("https://evil.example/x") + + +def test_literal_ip_hosts_are_refused(monkeypatch): + with pytest.raises(network.UnsafeDestinationError): + network.ensure_public_destination("http://127.0.0.1:8080/") + + +def test_one_private_answer_among_public_ones_is_refused(monkeypatch): + def fake_getaddrinfo(*_args, **_kwargs): + return [ + (None, None, None, None, ("93.184.216.34", 0)), + (None, None, None, None, ("10.0.0.5", 0)), + ] + + monkeypatch.setattr(network, "_system_getaddrinfo", fake_getaddrinfo) + + with pytest.raises(network.UnsafeDestinationError): + network.ensure_public_destination("https://mixed.example/") + + +def test_public_and_unresolvable_hosts_pass(monkeypatch): + _resolve_to(monkeypatch, {"ok.example": "93.184.216.34", "v6.example": "2606:2800:220:1::1"}) + + network.ensure_public_destination("https://ok.example/book") + network.ensure_public_destination("https://v6.example/book") + network.ensure_public_destination("https://does-not-resolve.example/book") + + +def test_other_schemes_are_refused(): + with pytest.raises(network.UnsafeDestinationError): + network.ensure_public_destination("file:///etc/passwd") + + +def test_configured_mirror_on_a_private_address_is_allowed(monkeypatch): + monkeypatch.setattr( + mirrors, "_get_config", lambda: {"OCEANOFPDF_MIRROR_URLS": ["http://lan-mirror.home"]} + ) + _resolve_to(monkeypatch, {"lan-mirror.home": "192.168.1.20"}) + + network.ensure_public_destination("http://lan-mirror.home/book") + + +def test_redirect_hook_checks_each_hop(monkeypatch): + _resolve_to(monkeypatch, {"ok.example": "93.184.216.34", "internal.example": "10.0.0.9"}) + hook = network.REDIRECT_GUARD_HOOKS["response"][0] + + def hop(location): + return SimpleNamespace( + is_redirect=True, url="https://ok.example/a", headers={"Location": location} + ) + + hook(hop("/b")) + with pytest.raises(network.UnsafeDestinationError): + hook(hop("http://internal.example/secret")) + with pytest.raises(network.UnsafeDestinationError): + hook(hop("http://127.0.0.1/")) + hook(SimpleNamespace(is_redirect=False, url="https://ok.example/a", headers={}))