diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index 1b6fbb37..f3af791f 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -939,7 +939,17 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None) result_path = ( Path(tempfile.gettempdir()) / f"shelfmark-bypass-{os.getpid()}-{time.time_ns()}.json" ) - payload = {"url": url, "retry": retry, "result_path": str(result_path)} + # DNS provider state lives only in the parent's memory (no disk persistence), so the + # freshly spawned helper would otherwise pre-resolve AA hostnames against the system + # resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's + # active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host + # resolver rules. + payload = { + "url": url, + "retry": retry, + "result_path": str(result_path), + "dns_config": network.get_dns_config(), + } env_vars = os.environ.copy() env_vars[_BYPASS_CHILD_ENV] = "1" env_vars = _prepare_child_browser_env(env_vars) @@ -1278,6 +1288,30 @@ def get_bypassed_page( return response_html +def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None: + """Mirror the parent process's active DNS provider in this helper subprocess. + + DNS state is in-memory only, so a fresh helper defaults to system DNS and would + pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver + that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on + the same DoH/custom resolver the parent already validated. + """ + provider = str(dns_config.get("provider") or "").strip().lower() + # "auto" means the parent has not rotated off system DNS yet, so the helper's own + # default initialization already matches it - nothing to override. + if not provider or provider == "auto": + return + manual_servers = dns_config.get("servers") if provider == "manual" else None + try: + network.set_dns_provider( + provider, + manual_servers, + use_doh=bool(dns_config.get("doh_enabled")), + ) + except (OSError, RuntimeError, ValueError) as exc: + logger.warning("Could not apply parent DNS config (%s): %s", provider, exc) + + def _run_child_process() -> int: """CLI entrypoint used by the Docker helper subprocess.""" request = json.loads(sys.stdin.read() or "{}") @@ -1287,6 +1321,10 @@ def _run_child_process() -> int: request.get("retry"), _coerce_positive_int(app_config.MAX_RETRY, 10) ) + dns_config = request.get("dns_config") + if isinstance(dns_config, dict): + _apply_parent_dns_config(dns_config) + try: html = get(url, retry=retry) payload = { diff --git a/shelfmark/config/booklore_settings.py b/shelfmark/config/booklore_settings.py index 0bcd60f9..f886ce7b 100644 --- a/shelfmark/config/booklore_settings.py +++ b/shelfmark/config/booklore_settings.py @@ -2,8 +2,8 @@ from __future__ import annotations -from typing import Any import hashlib +from typing import Any from shelfmark.core.config import config from shelfmark.core.logger import setup_logger diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index 14fe95b0..309104a5 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -852,6 +852,99 @@ def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver: return custom_resolver +# --- ISP / network DNS interference detection --------------------------------- +# Compare what the (tamperable) system resolver returns for a host against a +# tamper-resistant DoH lookup. Divergent answers are a strong signal the network is +# hijacking or NXDOMAIN-blocking the domain (a common reason AA downloads "work" but +# land on an ISP block page). Used to surface an actionable hint to the user. +_dns_interference_warned: set[str] = set() +_dns_interference_active = False + + +def _build_detection_doh_resolver() -> DoHResolver | None: + """Build a throwaway DoH resolver for interference checks (no socket patching). + + Honours the DoH provider the user selected (``DNS_PROVIDERS[_current_dns_index]``), + falling back to the first configured provider when none is active. The endpoint is + pinned to the provider's own nameserver IP so resolving the DoH host can't be + redirected by the very DNS layer the check is meant to detect. + """ + if 0 <= _current_dns_index < len(DNS_PROVIDERS): + _name, servers, doh_url = DNS_PROVIDERS[_current_dns_index] + elif DNS_PROVIDERS: + _name, servers, doh_url = DNS_PROVIDERS[0] + else: + return None + server_hostname = urllib.parse.urlparse(doh_url).hostname or "" + if not server_hostname or not servers: + return None + return DoHResolver(doh_url, server_hostname, servers[0]) + + +def detect_dns_interference(hostname: str) -> dict[str, list[str]] | None: + """Detect network DNS interference by comparing system DNS against DoH. + + Returns ``{"system_ips": [...], "doh_ips": [...]}`` when the two resolvers disagree + (no overlapping IPs), otherwise None. No-op for IP literals / local hostnames and + when DoH resolution is unavailable, so it never produces a false positive. + """ + host = (hostname or "").strip().lower() + if not host or _is_ip_address(host) or _is_local_address(host): + return None + resolver = _build_detection_doh_resolver() + if resolver is None: + return None + try: + system_ips = {str(info[4][0]) for info in original_getaddrinfo(host, 443, socket.AF_INET)} + except OSError: + return None + if not system_ips: + return None + doh_ips = {ip for ip in resolver.resolve(host, "A") if ip} + if not doh_ips or (system_ips & doh_ips): + return None + return {"system_ips": sorted(system_ips), "doh_ips": sorted(doh_ips)} + + +def note_possible_dns_interference(hostname: str) -> bool: + """Check ``hostname`` for DNS interference, logging an actionable warning once. + + Returns True when interference has been detected this session. The check runs at + most once per host to avoid repeated DoH lookups and log spam. + """ + global _dns_interference_active + host = (hostname or "").strip().lower() + if not host or host in _dns_interference_warned: + return _dns_interference_active + _dns_interference_warned.add(host) + + result = detect_dns_interference(host) + if not result: + return _dns_interference_active + + _dns_interference_active = True + routing_via_doh = _current_dns_index >= 0 and bool(DOH_SERVER) + remedy = ( + "Shelfmark is routing this domain through DNS-over-HTTPS to work around it." + if routing_via_doh + else "Enable DNS-over-HTTPS (USE_DOH=true) or set a custom DNS provider to bypass it." + ) + logger.warning( + "Possible ISP/network DNS interference for %s: system DNS resolves to %s but DoH " + "resolves to %s. The network appears to be blocking or redirecting this domain. %s", + host, + result["system_ips"], + result["doh_ips"], + remedy, + ) + return True + + +def dns_interference_detected() -> bool: + """Whether network DNS interference has been detected this session.""" + return _dns_interference_active + + def init_doh_resolver(doh_server: str = "") -> DoHResolver | None: """Initialize DNS over HTTPS resolver.""" server = doh_server or DOH_SERVER diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index 2d889002..bdc91983 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -1621,6 +1621,12 @@ def _extract_slow_download_url( link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]] logger.warning("No download URL found. First 10 links: %s", link_texts) + # A bypassed page with no AA download links often means the network served a wrong + # page (e.g. an ISP block page) instead of Anna's Archive. Probe for DNS interference + # so we can give the user an actionable hint instead of a generic failure. + host = urlparse(link).hostname or "" + if host: + network.note_possible_dns_interference(host) return "" @@ -2053,7 +2059,14 @@ class DirectDownloadHandler(DownloadHandler): return None if not success_url: - status_callback("error", "All download sources failed") + if network.dns_interference_detected(): + status_callback( + "error", + "All sources failed - your network/ISP appears to be blocking " + "Anna's Archive. Enable DNS-over-HTTPS in settings.", + ) + else: + status_callback("error", "All download sources failed") return None # Return temp path - orchestrator handles post-processing (archive extraction, ingest) diff --git a/tests/bypass/test_internal_bypasser.py b/tests/bypass/test_internal_bypasser.py index 95ada60e..9fab707e 100644 --- a/tests/bypass/test_internal_bypasser.py +++ b/tests/bypass/test_internal_bypasser.py @@ -274,6 +274,75 @@ def test_run_child_process_writes_failure_for_unexpected_exception(monkeypatch, assert "plain SeleniumBase startup failure" in result["traceback"] +def test_run_child_process_applies_parent_dns_config(monkeypatch, tmp_path): + """Regression test for issue #1028: the helper subprocess must mirror the parent's + DNS provider, otherwise it pre-resolves AA hostnames against (possibly hijacked) + system DNS and Chrome loads the wrong page.""" + import io + import json + + import shelfmark.bypass.internal_bypasser as internal_bypasser + + result_path = tmp_path / "result.json" + request = { + "url": "https://annas-archive.pk/slow_download/abc/0/0", + "retry": 1, + "result_path": str(result_path), + "dns_config": { + "provider": "cloudflare", + "servers": ["1.1.1.1", "1.0.0.1"], + "doh_url": "https://cloudflare-dns.com/dns-query", + "doh_enabled": True, + "is_auto_mode": True, + }, + } + + applied: list[tuple] = [] + monkeypatch.setattr( + internal_bypasser.network, + "set_dns_provider", + lambda provider, manual=None, *, use_doh=None: applied.append((provider, manual, use_doh)), + ) + monkeypatch.setattr(internal_bypasser, "get", lambda *_a, **_k: "ok") + monkeypatch.setattr(internal_bypasser.sys, "stdin", io.StringIO(json.dumps(request))) + + assert internal_bypasser._run_child_process() == 0 + assert applied == [("cloudflare", None, True)] + + +def test_apply_parent_dns_config_skips_auto_and_empty(monkeypatch): + import shelfmark.bypass.internal_bypasser as internal_bypasser + + calls: list = [] + monkeypatch.setattr( + internal_bypasser.network, + "set_dns_provider", + lambda *a, **k: calls.append((a, k)), + ) + + internal_bypasser._apply_parent_dns_config({"provider": "auto"}) + internal_bypasser._apply_parent_dns_config({}) + + assert calls == [] + + +def test_apply_parent_dns_config_forwards_manual_servers(monkeypatch): + import shelfmark.bypass.internal_bypasser as internal_bypasser + + calls: list = [] + monkeypatch.setattr( + internal_bypasser.network, + "set_dns_provider", + lambda provider, manual=None, *, use_doh=None: calls.append((provider, manual, use_doh)), + ) + + internal_bypasser._apply_parent_dns_config( + {"provider": "manual", "servers": ["9.9.9.9"], "doh_enabled": False} + ) + + assert calls == [("manual", ["9.9.9.9"], False)] + + def test_prepare_child_browser_env_uses_writable_runtime_paths(monkeypatch, tmp_path): import stat diff --git a/tests/download/test_network_dns_failover.py b/tests/download/test_network_dns_failover.py index 80a781ea..c8fe1f05 100644 --- a/tests/download/test_network_dns_failover.py +++ b/tests/download/test_network_dns_failover.py @@ -137,3 +137,135 @@ def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch): assert result == [ (network.socket.AF_INET, network.socket.SOCK_STREAM, 6, "", ("203.0.113.10", 443)) ] + + +def _addrinfo(ip): + return [(2, 1, 6, "", (ip, 443))] + + +class _FakeDoHResolver: + def __init__(self, ips): + self._ips = ips + + def resolve(self, _hostname, _record_type): + return list(self._ips) + + +def test_build_detection_doh_resolver_uses_selected_provider(monkeypatch): + import shelfmark.download.network as network + + monkeypatch.setattr( + network, + "DNS_PROVIDERS", + [ + ("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"), + ("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"), + ], + ) + monkeypatch.setattr(network, "_current_dns_index", 1) # user selected quad9 + + resolver = network._build_detection_doh_resolver() + + assert resolver is not None + assert resolver.base_url == "https://dns.quad9.net/dns-query" + assert resolver.hostname == "dns.quad9.net" + assert resolver.ip == "9.9.9.9" + + +def test_build_detection_doh_resolver_falls_back_to_first_provider(monkeypatch): + import shelfmark.download.network as network + + monkeypatch.setattr( + network, + "DNS_PROVIDERS", + [ + ("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"), + ("quad9", ["9.9.9.9"], "https://dns.quad9.net/dns-query"), + ], + ) + monkeypatch.setattr(network, "_current_dns_index", -1) # system / not yet rotated + + resolver = network._build_detection_doh_resolver() + + assert resolver is not None + assert resolver.base_url == "https://cloudflare-dns.com/dns-query" + assert resolver.ip == "1.1.1.1" + + +def test_detect_dns_interference_flags_divergent_resolvers(monkeypatch): + import shelfmark.download.network as network + + # System DNS (hijacked) returns an ISP block-page IP; DoH returns the real one. + monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1")) + monkeypatch.setattr( + network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"]) + ) + + result = network.detect_dns_interference("annas-archive.pk") + + assert result == {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]} + + +def test_detect_dns_interference_none_when_resolvers_agree(monkeypatch): + import shelfmark.download.network as network + + monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("203.0.113.7")) + monkeypatch.setattr( + network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"]) + ) + + assert network.detect_dns_interference("annas-archive.pk") is None + + +def test_detect_dns_interference_none_when_doh_unavailable(monkeypatch): + import shelfmark.download.network as network + + monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1")) + monkeypatch.setattr(network, "_build_detection_doh_resolver", lambda: None) + + assert network.detect_dns_interference("annas-archive.pk") is None + + +def test_detect_dns_interference_skips_ip_and_local(monkeypatch): + import shelfmark.download.network as network + + def _should_not_run(): + raise AssertionError("resolver should not be built for IP/local hosts") + + monkeypatch.setattr(network, "_build_detection_doh_resolver", _should_not_run) + + assert network.detect_dns_interference("1.2.3.4") is None + assert network.detect_dns_interference("localhost") is None + + +def test_note_possible_dns_interference_warns_once_and_sets_flag(monkeypatch): + import shelfmark.download.network as network + + monkeypatch.setattr(network, "_dns_interference_warned", set()) + monkeypatch.setattr(network, "_dns_interference_active", False) + + calls: list[str] = [] + monkeypatch.setattr( + network, + "detect_dns_interference", + lambda host: ( + calls.append(host) or {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]} + ), + ) + + assert network.note_possible_dns_interference("annas-archive.pk") is True + assert network.dns_interference_detected() is True + # A repeat check for the same host must not re-run the costly detection. + assert network.note_possible_dns_interference("annas-archive.pk") is True + assert calls == ["annas-archive.pk"] + + +def test_note_possible_dns_interference_no_detection_keeps_flag_false(monkeypatch): + import shelfmark.download.network as network + + monkeypatch.setattr(network, "_dns_interference_warned", set()) + monkeypatch.setattr(network, "_dns_interference_active", False) + monkeypatch.setattr(network, "detect_dns_interference", lambda _host: None) + + assert network.note_possible_dns_interference("annas-archive.pk") is False + assert network.dns_interference_detected() is False