Fix internal bypasser using system DNS in subprocess, breaking downloads behind ISP DNS blocks (#1028) (#1080)

shelfmark/bypass/internal_bypasser.py

@@ -939,7 +939,17 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
     result_path = (
         Path(tempfile.gettempdir()) / f"shelfmark-bypass-{os.getpid()}-{time.time_ns()}.json"
     )
-    payload = {"url": url, "retry": retry, "result_path": str(result_path)}
+    # DNS provider state lives only in the parent's memory (no disk persistence), so the
+    # freshly spawned helper would otherwise pre-resolve AA hostnames against the system
+    # resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
+    # active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
+    # resolver rules.
+    payload = {
+        "url": url,
+        "retry": retry,
+        "result_path": str(result_path),
+        "dns_config": network.get_dns_config(),
+    }
     env_vars = os.environ.copy()
     env_vars[_BYPASS_CHILD_ENV] = "1"
     env_vars = _prepare_child_browser_env(env_vars)
@@ -1278,6 +1288,30 @@ def get_bypassed_page(
     return response_html
 
 
+def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
+    """Mirror the parent process's active DNS provider in this helper subprocess.
+
+    DNS state is in-memory only, so a fresh helper defaults to system DNS and would
+    pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
+    that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
+    the same DoH/custom resolver the parent already validated.
+    """
+    provider = str(dns_config.get("provider") or "").strip().lower()
+    # "auto" means the parent has not rotated off system DNS yet, so the helper's own
+    # default initialization already matches it - nothing to override.
+    if not provider or provider == "auto":
+        return
+    manual_servers = dns_config.get("servers") if provider == "manual" else None
+    try:
+        network.set_dns_provider(
+            provider,
+            manual_servers,
+            use_doh=bool(dns_config.get("doh_enabled")),
+        )
+    except (OSError, RuntimeError, ValueError) as exc:
+        logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
+
+
 def _run_child_process() -> int:
     """CLI entrypoint used by the Docker helper subprocess."""
     request = json.loads(sys.stdin.read() or "{}")
@@ -1287,6 +1321,10 @@ def _run_child_process() -> int:
         request.get("retry"), _coerce_positive_int(app_config.MAX_RETRY, 10)
     )
 
+    dns_config = request.get("dns_config")
+    if isinstance(dns_config, dict):
+        _apply_parent_dns_config(dns_config)
+
     try:
         html = get(url, retry=retry)
         payload = {

shelfmark/config/booklore_settings.py

@@ -2,8 +2,8 @@
 
 from __future__ import annotations
 
-from typing import Any
 import hashlib
+from typing import Any
 
 from shelfmark.core.config import config
 from shelfmark.core.logger import setup_logger

shelfmark/download/network.py

@@ -852,6 +852,99 @@ def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver:
     return custom_resolver
 
 
+# --- ISP / network DNS interference detection ---------------------------------
+# Compare what the (tamperable) system resolver returns for a host against a
+# tamper-resistant DoH lookup. Divergent answers are a strong signal the network is
+# hijacking or NXDOMAIN-blocking the domain (a common reason AA downloads "work" but
+# land on an ISP block page). Used to surface an actionable hint to the user.
+_dns_interference_warned: set[str] = set()
+_dns_interference_active = False
+
+
+def _build_detection_doh_resolver() -> DoHResolver | None:
+    """Build a throwaway DoH resolver for interference checks (no socket patching).
+
+    Honours the DoH provider the user selected (``DNS_PROVIDERS[_current_dns_index]``),
+    falling back to the first configured provider when none is active. The endpoint is
+    pinned to the provider's own nameserver IP so resolving the DoH host can't be
+    redirected by the very DNS layer the check is meant to detect.
+    """
+    if 0 <= _current_dns_index < len(DNS_PROVIDERS):
+        _name, servers, doh_url = DNS_PROVIDERS[_current_dns_index]
+    elif DNS_PROVIDERS:
+        _name, servers, doh_url = DNS_PROVIDERS[0]
+    else:
+        return None
+    server_hostname = urllib.parse.urlparse(doh_url).hostname or ""
+    if not server_hostname or not servers:
+        return None
+    return DoHResolver(doh_url, server_hostname, servers[0])
+
+
+def detect_dns_interference(hostname: str) -> dict[str, list[str]] | None:
+    """Detect network DNS interference by comparing system DNS against DoH.
+
+    Returns ``{"system_ips": [...], "doh_ips": [...]}`` when the two resolvers disagree
+    (no overlapping IPs), otherwise None. No-op for IP literals / local hostnames and
+    when DoH resolution is unavailable, so it never produces a false positive.
+    """
+    host = (hostname or "").strip().lower()
+    if not host or _is_ip_address(host) or _is_local_address(host):
+        return None
+    resolver = _build_detection_doh_resolver()
+    if resolver is None:
+        return None
+    try:
+        system_ips = {str(info[4][0]) for info in original_getaddrinfo(host, 443, socket.AF_INET)}
+    except OSError:
+        return None
+    if not system_ips:
+        return None
+    doh_ips = {ip for ip in resolver.resolve(host, "A") if ip}
+    if not doh_ips or (system_ips & doh_ips):
+        return None
+    return {"system_ips": sorted(system_ips), "doh_ips": sorted(doh_ips)}
+
+
+def note_possible_dns_interference(hostname: str) -> bool:
+    """Check ``hostname`` for DNS interference, logging an actionable warning once.
+
+    Returns True when interference has been detected this session. The check runs at
+    most once per host to avoid repeated DoH lookups and log spam.
+    """
+    global _dns_interference_active
+    host = (hostname or "").strip().lower()
+    if not host or host in _dns_interference_warned:
+        return _dns_interference_active
+    _dns_interference_warned.add(host)
+
+    result = detect_dns_interference(host)
+    if not result:
+        return _dns_interference_active
+
+    _dns_interference_active = True
+    routing_via_doh = _current_dns_index >= 0 and bool(DOH_SERVER)
+    remedy = (
+        "Shelfmark is routing this domain through DNS-over-HTTPS to work around it."
+        if routing_via_doh
+        else "Enable DNS-over-HTTPS (USE_DOH=true) or set a custom DNS provider to bypass it."
+    )
+    logger.warning(
+        "Possible ISP/network DNS interference for %s: system DNS resolves to %s but DoH "
+        "resolves to %s. The network appears to be blocking or redirecting this domain. %s",
+        host,
+        result["system_ips"],
+        result["doh_ips"],
+        remedy,
+    )
+    return True
+
+
+def dns_interference_detected() -> bool:
+    """Whether network DNS interference has been detected this session."""
+    return _dns_interference_active
+
+
 def init_doh_resolver(doh_server: str = "") -> DoHResolver | None:
     """Initialize DNS over HTTPS resolver."""
     server = doh_server or DOH_SERVER

shelfmark/release_sources/direct_download.py

@@ -1621,6 +1621,12 @@ def _extract_slow_download_url(
 
     link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
     logger.warning("No download URL found. First 10 links: %s", link_texts)
+    # A bypassed page with no AA download links often means the network served a wrong
+    # page (e.g. an ISP block page) instead of Anna's Archive. Probe for DNS interference
+    # so we can give the user an actionable hint instead of a generic failure.
+    host = urlparse(link).hostname or ""
+    if host:
+        network.note_possible_dns_interference(host)
     return ""
 
 
@@ -2053,7 +2059,14 @@ class DirectDownloadHandler(DownloadHandler):
                 return None
 
             if not success_url:
-                status_callback("error", "All download sources failed")
+                if network.dns_interference_detected():
+                    status_callback(
+                        "error",
+                        "All sources failed - your network/ISP appears to be blocking "
+                        "Anna's Archive. Enable DNS-over-HTTPS in settings.",
+                    )
+                else:
+                    status_callback("error", "All download sources failed")
                 return None
 
             # Return temp path - orchestrator handles post-processing (archive extraction, ingest)

tests/bypass/test_internal_bypasser.py

@@ -274,6 +274,75 @@ def test_run_child_process_writes_failure_for_unexpected_exception(monkeypatch,
     assert "plain SeleniumBase startup failure" in result["traceback"]
 
 
+def test_run_child_process_applies_parent_dns_config(monkeypatch, tmp_path):
+    """Regression test for issue #1028: the helper subprocess must mirror the parent's
+    DNS provider, otherwise it pre-resolves AA hostnames against (possibly hijacked)
+    system DNS and Chrome loads the wrong page."""
+    import io
+    import json
+
+    import shelfmark.bypass.internal_bypasser as internal_bypasser
+
+    result_path = tmp_path / "result.json"
+    request = {
+        "url": "https://annas-archive.pk/slow_download/abc/0/0",
+        "retry": 1,
+        "result_path": str(result_path),
+        "dns_config": {
+            "provider": "cloudflare",
+            "servers": ["1.1.1.1", "1.0.0.1"],
+            "doh_url": "https://cloudflare-dns.com/dns-query",
+            "doh_enabled": True,
+            "is_auto_mode": True,
+        },
+    }
+
+    applied: list[tuple] = []
+    monkeypatch.setattr(
+        internal_bypasser.network,
+        "set_dns_provider",
+        lambda provider, manual=None, *, use_doh=None: applied.append((provider, manual, use_doh)),
+    )
+    monkeypatch.setattr(internal_bypasser, "get", lambda *_a, **_k: "<html>ok</html>")
+    monkeypatch.setattr(internal_bypasser.sys, "stdin", io.StringIO(json.dumps(request)))
+
+    assert internal_bypasser._run_child_process() == 0
+    assert applied == [("cloudflare", None, True)]
+
+
+def test_apply_parent_dns_config_skips_auto_and_empty(monkeypatch):
+    import shelfmark.bypass.internal_bypasser as internal_bypasser
+
+    calls: list = []
+    monkeypatch.setattr(
+        internal_bypasser.network,
+        "set_dns_provider",
+        lambda *a, **k: calls.append((a, k)),
+    )
+
+    internal_bypasser._apply_parent_dns_config({"provider": "auto"})
+    internal_bypasser._apply_parent_dns_config({})
+
+    assert calls == []
+
+
+def test_apply_parent_dns_config_forwards_manual_servers(monkeypatch):
+    import shelfmark.bypass.internal_bypasser as internal_bypasser
+
+    calls: list = []
+    monkeypatch.setattr(
+        internal_bypasser.network,
+        "set_dns_provider",
+        lambda provider, manual=None, *, use_doh=None: calls.append((provider, manual, use_doh)),
+    )
+
+    internal_bypasser._apply_parent_dns_config(
+        {"provider": "manual", "servers": ["9.9.9.9"], "doh_enabled": False}
+    )
+
+    assert calls == [("manual", ["9.9.9.9"], False)]
+
+
 def test_prepare_child_browser_env_uses_writable_runtime_paths(monkeypatch, tmp_path):
     import stat
 

tests/download/test_network_dns_failover.py

@@ -137,3 +137,135 @@ def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch):
     assert result == [
         (network.socket.AF_INET, network.socket.SOCK_STREAM, 6, "", ("203.0.113.10", 443))
     ]
+
+
+def _addrinfo(ip):
+    return [(2, 1, 6, "", (ip, 443))]
+
+
+class _FakeDoHResolver:
+    def __init__(self, ips):
+        self._ips = ips
+
+    def resolve(self, _hostname, _record_type):
+        return list(self._ips)
+
+
+def test_build_detection_doh_resolver_uses_selected_provider(monkeypatch):
+    import shelfmark.download.network as network
+
+    monkeypatch.setattr(
+        network,
+        "DNS_PROVIDERS",
+        [
+            ("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
+            ("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"),
+        ],
+    )
+    monkeypatch.setattr(network, "_current_dns_index", 1)  # user selected quad9
+
+    resolver = network._build_detection_doh_resolver()
+
+    assert resolver is not None
+    assert resolver.base_url == "https://dns.quad9.net/dns-query"
+    assert resolver.hostname == "dns.quad9.net"
+    assert resolver.ip == "9.9.9.9"
+
+
+def test_build_detection_doh_resolver_falls_back_to_first_provider(monkeypatch):
+    import shelfmark.download.network as network
+
+    monkeypatch.setattr(
+        network,
+        "DNS_PROVIDERS",
+        [
+            ("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
+            ("quad9", ["9.9.9.9"], "https://dns.quad9.net/dns-query"),
+        ],
+    )
+    monkeypatch.setattr(network, "_current_dns_index", -1)  # system / not yet rotated
+
+    resolver = network._build_detection_doh_resolver()
+
+    assert resolver is not None
+    assert resolver.base_url == "https://cloudflare-dns.com/dns-query"
+    assert resolver.ip == "1.1.1.1"
+
+
+def test_detect_dns_interference_flags_divergent_resolvers(monkeypatch):
+    import shelfmark.download.network as network
+
+    # System DNS (hijacked) returns an ISP block-page IP; DoH returns the real one.
+    monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
+    monkeypatch.setattr(
+        network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
+    )
+
+    result = network.detect_dns_interference("annas-archive.pk")
+
+    assert result == {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
+
+
+def test_detect_dns_interference_none_when_resolvers_agree(monkeypatch):
+    import shelfmark.download.network as network
+
+    monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("203.0.113.7"))
+    monkeypatch.setattr(
+        network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
+    )
+
+    assert network.detect_dns_interference("annas-archive.pk") is None
+
+
+def test_detect_dns_interference_none_when_doh_unavailable(monkeypatch):
+    import shelfmark.download.network as network
+
+    monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
+    monkeypatch.setattr(network, "_build_detection_doh_resolver", lambda: None)
+
+    assert network.detect_dns_interference("annas-archive.pk") is None
+
+
+def test_detect_dns_interference_skips_ip_and_local(monkeypatch):
+    import shelfmark.download.network as network
+
+    def _should_not_run():
+        raise AssertionError("resolver should not be built for IP/local hosts")
+
+    monkeypatch.setattr(network, "_build_detection_doh_resolver", _should_not_run)
+
+    assert network.detect_dns_interference("1.2.3.4") is None
+    assert network.detect_dns_interference("localhost") is None
+
+
+def test_note_possible_dns_interference_warns_once_and_sets_flag(monkeypatch):
+    import shelfmark.download.network as network
+
+    monkeypatch.setattr(network, "_dns_interference_warned", set())
+    monkeypatch.setattr(network, "_dns_interference_active", False)
+
+    calls: list[str] = []
+    monkeypatch.setattr(
+        network,
+        "detect_dns_interference",
+        lambda host: (
+            calls.append(host) or {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
+        ),
+    )
+
+    assert network.note_possible_dns_interference("annas-archive.pk") is True
+    assert network.dns_interference_detected() is True
+    # A repeat check for the same host must not re-run the costly detection.
+    assert network.note_possible_dns_interference("annas-archive.pk") is True
+    assert calls == ["annas-archive.pk"]
+
+
+def test_note_possible_dns_interference_no_detection_keeps_flag_false(monkeypatch):
+    import shelfmark.download.network as network
+
+    monkeypatch.setattr(network, "_dns_interference_warned", set())
+    monkeypatch.setattr(network, "_dns_interference_active", False)
+    monkeypatch.setattr(network, "detect_dns_interference", lambda _host: None)
+
+    assert network.note_possible_dns_interference("annas-archive.pk") is False
+    assert network.dns_interference_detected() is False