Merge commit from fork

A signed-in user picks the release fields of POST /api/releases/download, and a record
id with the "oceanofpdf:" prefix reached the provider without handles() or is_enabled().
The provider then fetched source_url, POSTed to the form action found on that page and
fetched the meta-refresh target, with no check on any of them.

The provider now refuses to run when it is switched off, and the book page, the form
action and every POST redirect hop must be on the configured mirror (same scheme, host
and port). The signed file link may sit on another host but must be http(s).

network.ensure_public_destination refuses a host that resolves to a loopback, private,
link-local or otherwise non-global address (IPv4-mapped IPv6 judged by its IPv4 address).
Hosts the operator configured as mirrors and names that do not resolve pass as before.
html_get_page and download_url call it on their first URL and check each redirect hop
through a response hook. The image cache uses the shared address check too.

Co-authored-by: CaliBrain <calibrain@l4n.xyz>

shelfmark/core/image_cache.py

@@ -3,7 +3,6 @@
 import hashlib
 import ipaddress
 import json
-import socket
 import threading
 import time
 from http import HTTPStatus
@@ -15,7 +14,7 @@ import requests
 
 from shelfmark.core.logger import setup_logger
 from shelfmark.core.request_helpers import coerce_int
-from shelfmark.download.network import get_ssl_verify
+from shelfmark.download.network import get_ssl_verify, hostname_resolves_publicly
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -526,13 +525,7 @@ class ImageCacheService:
         if not hostname:
             return None
 
-        try:
-            resolved = socket.getaddrinfo(hostname, None)
-            for _, _, _, _, sockaddr in resolved:
-                ip = ipaddress.ip_address(sockaddr[0])
-                if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
-                    return None
-        except socket.gaierror, ValueError:
+        if not hostname_resolves_publicly(hostname, use_custom_dns=True):
             return None
 
         return prepared_url

shelfmark/download/http.py

@@ -518,6 +518,11 @@ def html_get_page(
     )
     original_url = url
     current_url = selector.rewrite(original_url)
+    try:
+        network.ensure_public_destination(current_url)
+    except network.UnsafeDestinationError as e:
+        logger.warning("Blocked page fetch: %s", e)
+        return _fail("The page address is not a public host.", current_url)
     use_bypasser_now = use_bypasser
     # Survives across attempts so a cookie won once is still presented on later retries.
     handshake_cookies: dict[str, str] = {}
@@ -572,6 +577,7 @@ def html_get_page(
                     headers=headers,
                     allow_redirects=allow_redirects,
                     verify=get_ssl_verify(current_url),
+                    hooks=network.REDIRECT_GUARD_HOOKS,
                 )
 
                 # Z-Library gates the first hit with a 503 that carries nothing but a
@@ -879,6 +885,12 @@ def download_url(
     """Download content from URL with automatic retry and resume support."""
     selector = _selector or network.AAMirrorSelector()
     current_url = selector.rewrite(link)
+    try:
+        network.ensure_public_destination(current_url)
+    except network.UnsafeDestinationError as e:
+        logger.warning("Blocked download: %s", e)
+        return None
+    network.ensure_public_destination(current_url)
 
     # Build headers with optional referer
     headers = DOWNLOAD_HEADERS.copy()
@@ -914,6 +926,7 @@ def download_url(
                 cookies=cookies,
                 headers=headers,
                 verify=get_ssl_verify(current_url),
+                hooks=network.REDIRECT_GUARD_HOOKS,
             )
             response.raise_for_status()
 
@@ -1077,6 +1090,7 @@ def _try_resume(
                 headers=resume_headers,
                 cookies=cookies,
                 verify=get_ssl_verify(url),
+                hooks=network.REDIRECT_GUARD_HOOKS,
             )
 
             # Check resume support

shelfmark/download/network.py

@@ -549,6 +549,91 @@ def _is_local_address(host_str: str) -> bool:
         return addr.is_private or addr.is_loopback or addr.is_link_local
 
 
+class UnsafeDestinationError(requests.exceptions.InvalidURL):
+    """A request target resolves to a loopback, private or otherwise non-public address."""
+
+
+def _address_is_public(address: str) -> bool:
+    ip = ipaddress.ip_address(address.split("%", 1)[0])
+    if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+        ip = ip.ipv4_mapped
+    return ip.is_global
+
+
+def _system_getaddrinfo(hostname: str, port: None) -> list[Any]:
+    return original_getaddrinfo(hostname, port)
+
+
+def hostname_resolves_publicly(
+    hostname: str, *, allow_unresolved: bool = False, use_custom_dns: bool = False
+) -> bool:
+    """Return True when every address ``hostname`` resolves to is globally routable.
+
+    A name that does not resolve counts as public only with ``allow_unresolved``, so the
+    caller's own request fails on it the way it always did. The check asks the system
+    resolver unless ``use_custom_dns`` is set: a failed lookup through the patched
+    resolver switches the DNS provider, and a pre-flight check must not do that.
+    """
+    resolver = socket.getaddrinfo if use_custom_dns else _system_getaddrinfo
+    try:
+        return _address_is_public(hostname)
+    except ValueError:
+        pass  # not an IP literal, resolve it
+    try:
+        resolved = resolver(hostname, None)
+        return all(_address_is_public(str(sockaddr[0])) for *_, sockaddr in resolved)
+    except socket.gaierror:
+        return allow_unresolved
+    except ValueError:
+        return False
+
+
+def _configured_mirror_hosts() -> set[str]:
+    """Hostnames of every mirror the operator configured, which are trusted as given."""
+    from shelfmark.core import mirrors
+
+    urls = [
+        *mirrors.get_aa_mirrors(),
+        *mirrors.get_libgen_mirrors(),
+        *mirrors.get_zlib_mirrors(),
+        *mirrors.get_welib_mirrors(),
+        *mirrors.get_oceanofpdf_mirrors(),
+    ]
+    return {(urllib.parse.urlparse(url).hostname or "").lower() for url in urls}
+
+
+def ensure_public_destination(url: str) -> None:
+    """Refuse a URL whose host resolves to a non-public address.
+
+    Direct-download pages and file links come from remote sites or from API callers, so a
+    hostile value could otherwise aim the server at loopback, the LAN or a sibling
+    container. A host the operator configured as a mirror is allowed as written, and so is
+    a name that does not resolve (the request fails on it normally).
+    """
+    parsed = urllib.parse.urlparse(url)
+    hostname = (parsed.hostname or "").lower().rstrip(".")
+    if parsed.scheme not in ("http", "https") or not hostname:
+        msg = f"Refusing to request {url!r}: not an http(s) URL with a host"
+        raise UnsafeDestinationError(msg)
+    if hostname in _configured_mirror_hosts():
+        return
+    if not hostname_resolves_publicly(hostname, allow_unresolved=True):
+        msg = f"Refusing to request {hostname!r}: it resolves to a non-public address"
+        raise UnsafeDestinationError(msg)
+
+
+def _guard_redirect_hop(response: requests.Response, *_args: object, **_kwargs: object) -> None:
+    """Response hook: check where a redirect points before requests follows it."""
+    if response.is_redirect:
+        location = response.headers.get("Location", "")
+        if location:
+            ensure_public_destination(urllib.parse.urljoin(response.url, location))
+
+
+# Pass as ``hooks=`` so every redirect hop of a request is checked, not just its first URL.
+REDIRECT_GUARD_HOOKS: dict[str, Any] = {"response": [_guard_redirect_hop]}
+
+
 def _is_ip_address(host_str: str) -> bool:
     """Check if a string is a valid IP address (IPv4 or IPv6)."""
     try:

shelfmark/release_sources/direct_download/oceanofpdf.py

@@ -67,6 +67,9 @@ _HTML_SIGNATURES = (b"<!doctype", b"<html")
 # changed page park a download for minutes.
 _MAX_REDIRECT_DELAY_SECONDS = 10
 _POST_TIMEOUT = (10, 30)
+_MAX_POST_REDIRECTS = 5
+_HTTP_TEMP_REDIRECT = 307
+_HTTP_PERM_REDIRECT = 308
 _HTTP_FORBIDDEN = 403
 _HTTP_SERVICE_UNAVAILABLE = 503
 
@@ -94,6 +97,27 @@ def _is_switched_on() -> bool:
     return False
 
 
+def _is_site_url(url: str) -> bool:
+    """Whether ``url`` is a page of the configured OceanofPDF mirror itself.
+
+    The book page, the download form's action and any redirect between them must stay on
+    the mirror, same scheme and port, so a record or a page cannot point the fetch at
+    another host. The signed file link is the one URL that legitimately sits elsewhere.
+    """
+    base_url = _base_url()
+    if not base_url:
+        return False
+    try:
+        parsed, base = urlparse(url), urlparse(base_url)
+        return (
+            parsed.scheme == base.scheme
+            and (parsed.hostname or "").lower().rstrip(".") == (base.hostname or "").lower()
+            and parsed.port == base.port
+        )
+    except ValueError:
+        return False
+
+
 def _search_url(base_url: str, query: str) -> str:
     return f"{base_url}/?s={quote_plus(query)}"
 
@@ -233,15 +257,34 @@ def _post_fetch_form(action_url: str, fields: dict[str, str], referer: str) -> r
         "Referer": referer,
         "Origin": f"{urlparse(referer).scheme}://{urlparse(referer).netloc}",
     }
-    return requests.post(
-        action_url,
-        data=fields,
-        headers=headers,
-        cookies=downloader.get_cf_cookies_for_domain(hostname),
-        timeout=_POST_TIMEOUT,
-        proxies=network.get_proxies(action_url),
-        verify=network.get_ssl_verify(action_url),
-    )
+    method = "POST"
+    url = action_url
+    for _ in range(_MAX_POST_REDIRECTS + 1):
+        if not _is_site_url(url):
+            msg = f"OceanofPDF download form points off the mirror: {url}"
+            raise requests.exceptions.InvalidURL(msg)
+        network.ensure_public_destination(url)
+        common = {
+            "headers": headers,
+            "cookies": downloader.get_cf_cookies_for_domain(hostname),
+            "timeout": _POST_TIMEOUT,
+            "proxies": network.get_proxies(url),
+            "verify": network.get_ssl_verify(url),
+            "allow_redirects": False,
+        }
+        response = (
+            requests.post(url, data=fields, **common)
+            if method == "POST"
+            else requests.get(url, **common)
+        )
+        location = response.headers.get("Location") if response.is_redirect else None
+        if not location:
+            return response
+        url = urljoin(url, location)
+        if response.status_code not in (_HTTP_TEMP_REDIRECT, _HTTP_PERM_REDIRECT):
+            method = "GET"
+    msg = "OceanofPDF download form redirected too many times"
+    raise requests.exceptions.TooManyRedirects(msg)
 
 
 def _log_unresolved_fetch_page(response: requests.Response) -> None:
@@ -404,6 +447,12 @@ class OceanofPDFProvider:
         if not book_url:
             msg = f"OceanofPDF record {book_info.id!r} has no book page URL"
             raise RuntimeError(msg)
+        if not self.is_enabled():
+            msg = "OceanofPDF is not enabled"
+            raise RuntimeError(msg)
+        if not _is_site_url(book_url):
+            msg = f"OceanofPDF record {book_info.id!r} does not point at the configured mirror"
+            raise RuntimeError(msg)
 
         def status(message: str) -> None:
             if status_callback:
@@ -432,6 +481,9 @@ class OceanofPDFProvider:
                 return None
             action, fields = form
             action_url = urljoin(book_url, action)
+            if not _is_site_url(action_url):
+                logger.warning("OceanofPDF download form points off the mirror: %s", action_url)
+                return None
 
             if cancel_flag and cancel_flag.is_set():
                 return None
@@ -441,6 +493,9 @@ class OceanofPDFProvider:
                 logger.warning("OceanofPDF did not return a file link for %s", book_url)
                 return None
             file_url, delay = resolved
+            if urlparse(file_url).scheme not in ("http", "https"):
+                logger.warning("OceanofPDF file link is not an http(s) URL: %s", file_url)
+                return None
 
             wait = min(delay, _MAX_REDIRECT_DELAY_SECONDS)
             if wait > 0:

tests/conftest.py

@@ -39,6 +39,24 @@ def _clear_torrent_fetch_cache():
     clear_torrent_fetch_cache()
 
 
+@pytest.fixture(autouse=True)
+def _public_destination_dns(monkeypatch):
+    """Resolve every host to a public address for the destination guard.
+
+    The guard looks hosts up before a request. Tests use made-up and real hostnames and
+    must not depend on the DNS of the machine running them (a filtering resolver answers
+    ``::`` for some mirrors), so the guard's resolver is stubbed. Tests of the guard
+    itself replace it again.
+    """
+    from shelfmark.download import network
+
+    monkeypatch.setattr(
+        network,
+        "_system_getaddrinfo",
+        lambda *_args, **_kwargs: [(2, 1, 6, "", ("93.184.216.34", 0))],
+    )
+
+
 @pytest.fixture
 def sample_prowlarr_result():
     """Sample Prowlarr API search result."""

tests/core/test_image_cache.py

@@ -54,7 +54,7 @@ def test_fetch_and_cache_blocks_unsafe_redirect(tmp_path, monkeypatch) -> None:
         calls.append(url)
         return RedirectResponse()
 
-    monkeypatch.setattr("shelfmark.core.image_cache.socket.getaddrinfo", fake_getaddrinfo)
+    monkeypatch.setattr("shelfmark.download.network.socket.getaddrinfo", fake_getaddrinfo)
     monkeypatch.setattr("shelfmark.core.image_cache.requests.get", fake_get)
 
     assert cache.fetch_and_cache("cover-redirect", "https://example.com/cover.jpg") is None

tests/direct_download/test_oceanofpdf.py

@@ -336,6 +336,8 @@ class _FakeResponse:
         self.text = text
         self.status_code = status_code
         self.url = url
+        self.headers: dict[str, str] = {}
+        self.is_redirect = False
 
     def raise_for_status(self):
         if self.status_code >= 400:
@@ -577,3 +579,99 @@ def test_mirror_list_is_normalised_and_first_entry_wins(monkeypatch):
 
     assert mirrors.get_oceanofpdf_mirrors() == ["https://ocean.example", "https://backup.example"]
     assert mirrors.get_oceanofpdf_base_url() == "https://ocean.example"
+
+
+# --- Hostile records and pages ---
+
+
+def _hostile_record(url: str) -> BrowseRecord:
+    return BrowseRecord(
+        id="oceanofpdf:x",
+        title="Dune",
+        source="direct_download",
+        format="epub",
+        source_url=url,
+    )
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://127.0.0.1:8080/admin",
+        "http://169.254.169.254/latest/meta-data/",
+        "https://other.example/book/",
+        "http://ocean.example/authors/dune/",
+        "https://ocean.example:8443/authors/dune/",
+        "file:///etc/passwd",
+    ],
+)
+def test_download_never_fetches_a_source_url_off_the_mirror(configured, monkeypatch, tmp_path, url):
+    requested = _serve_pages(monkeypatch, {url: BOOK_PAGE})
+    posts, downloads = _wire_download(monkeypatch, post_responses=[], payload=b"PK" * 10_000)
+
+    with pytest.raises(RuntimeError):
+        op.OceanofPDFProvider().download(
+            _hostile_record(url), tmp_path / "x.epub", None, Event(), None
+        )
+
+    assert url not in requested
+    assert posts == []
+    assert downloads == []
+
+
+def test_download_refuses_when_the_provider_is_switched_off(configured, monkeypatch, tmp_path):
+    configured["SOURCE_PRIORITY"] = [{"id": "oceanofpdf", "enabled": False}]
+    requested = _serve_pages(monkeypatch, {DUNE_URL: BOOK_PAGE})
+
+    with pytest.raises(RuntimeError):
+        op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None)
+
+    assert requested == []
+
+
+@pytest.mark.parametrize(
+    "action",
+    ["http://127.0.0.1:9000/Fetching_Resource.php", "https://evil.example/Fetching_Resource.php"],
+)
+def test_download_never_posts_to_a_form_action_off_the_mirror(
+    configured, monkeypatch, tmp_path, action
+):
+    page = BOOK_PAGE.replace("https://ocean.example/Fetching_Resource.php", action)
+    posts, downloads = _wire_download(monkeypatch, post_responses=[], payload=b"PK" * 10_000)
+    _serve_pages(monkeypatch, {DUNE_URL: page})
+
+    result = op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None)
+
+    assert result is None
+    assert posts == []
+    assert downloads == []
+
+
+def test_download_never_fetches_a_file_link_that_is_not_http(configured, monkeypatch, tmp_path):
+    interstitial = _page('<meta http-equiv="refresh" content="0; url=file:///etc/passwd">')
+    _posts, downloads = _wire_download(
+        monkeypatch, post_responses=[_FakeResponse(interstitial)], payload=b"PK" * 10_000
+    )
+
+    result = op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None)
+
+    assert result is None
+    assert downloads == []
+
+
+def test_form_post_redirect_off_the_mirror_is_not_followed(configured, monkeypatch, tmp_path):
+    redirect = _FakeResponse("", status_code=302)
+    redirect.is_redirect = True
+    redirect.headers = {"Location": "http://127.0.0.1:9000/internal"}
+    posts, downloads = _wire_download(
+        monkeypatch, post_responses=[redirect], payload=b"PK" * 10_000
+    )
+    gets: list[str] = []
+    monkeypatch.setattr(op.requests, "get", lambda url, **_kw: gets.append(url))
+
+    result = op.OceanofPDFProvider().download(_record(), tmp_path / "x.epub", None, Event(), None)
+
+    assert result is None
+    assert len(posts) == 1
+    assert gets == []
+    assert downloads == []

tests/download/test_network_destination_guard.py

@@ -0,0 +1,103 @@
+"""The destination guard behind the direct-download fetchers."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from shelfmark.core import mirrors
+from shelfmark.download import network
+
+
+def _resolve_to(monkeypatch, table: dict[str, str]) -> None:
+    def fake_getaddrinfo(host, *_args, **_kwargs):
+        if host not in table:
+            raise network.socket.gaierror(-2, "Name or service not known")
+        return [(None, None, None, None, (table[host], 0))]
+
+    monkeypatch.setattr(network, "_system_getaddrinfo", fake_getaddrinfo)
+
+
+@pytest.fixture(autouse=True)
+def no_mirrors(monkeypatch):
+    monkeypatch.setattr(mirrors, "_get_config", lambda: {})
+
+
+@pytest.mark.parametrize(
+    "address",
+    [
+        "127.0.0.1",
+        "10.1.2.3",
+        "172.17.0.2",
+        "192.168.1.5",
+        "169.254.169.254",
+        "100.64.0.1",
+        "0.0.0.0",
+        "::1",
+        "fe80::1",
+        "fd00::1",
+        "::ffff:127.0.0.1",
+        "::ffff:10.0.0.1",
+    ],
+)
+def test_non_public_addresses_are_refused(monkeypatch, address):
+    _resolve_to(monkeypatch, {"evil.example": address})
+
+    with pytest.raises(network.UnsafeDestinationError):
+        network.ensure_public_destination("https://evil.example/x")
+
+
+def test_literal_ip_hosts_are_refused(monkeypatch):
+    with pytest.raises(network.UnsafeDestinationError):
+        network.ensure_public_destination("http://127.0.0.1:8080/")
+
+
+def test_one_private_answer_among_public_ones_is_refused(monkeypatch):
+    def fake_getaddrinfo(*_args, **_kwargs):
+        return [
+            (None, None, None, None, ("93.184.216.34", 0)),
+            (None, None, None, None, ("10.0.0.5", 0)),
+        ]
+
+    monkeypatch.setattr(network, "_system_getaddrinfo", fake_getaddrinfo)
+
+    with pytest.raises(network.UnsafeDestinationError):
+        network.ensure_public_destination("https://mixed.example/")
+
+
+def test_public_and_unresolvable_hosts_pass(monkeypatch):
+    _resolve_to(monkeypatch, {"ok.example": "93.184.216.34", "v6.example": "2606:2800:220:1::1"})
+
+    network.ensure_public_destination("https://ok.example/book")
+    network.ensure_public_destination("https://v6.example/book")
+    network.ensure_public_destination("https://does-not-resolve.example/book")
+
+
+def test_other_schemes_are_refused():
+    with pytest.raises(network.UnsafeDestinationError):
+        network.ensure_public_destination("file:///etc/passwd")
+
+
+def test_configured_mirror_on_a_private_address_is_allowed(monkeypatch):
+    monkeypatch.setattr(
+        mirrors, "_get_config", lambda: {"OCEANOFPDF_MIRROR_URLS": ["http://lan-mirror.home"]}
+    )
+    _resolve_to(monkeypatch, {"lan-mirror.home": "192.168.1.20"})
+
+    network.ensure_public_destination("http://lan-mirror.home/book")
+
+
+def test_redirect_hook_checks_each_hop(monkeypatch):
+    _resolve_to(monkeypatch, {"ok.example": "93.184.216.34", "internal.example": "10.0.0.9"})
+    hook = network.REDIRECT_GUARD_HOOKS["response"][0]
+
+    def hop(location):
+        return SimpleNamespace(
+            is_redirect=True, url="https://ok.example/a", headers={"Location": location}
+        )
+
+    hook(hop("/b"))
+    with pytest.raises(network.UnsafeDestinationError):
+        hook(hop("http://internal.example/secret"))
+    with pytest.raises(network.UnsafeDestinationError):
+        hook(hop("http://127.0.0.1/"))
+    hook(SimpleNamespace(is_redirect=False, url="https://ok.example/a", headers={}))