From b656f019be4c406b595d644e2fd24812c6e49d97 Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Sat, 15 Aug 2026 14:17:44 -0400 Subject: [PATCH] feat(download): add DoH wireformat support, mirror quarantine, and search warmup (#1218) - Add RFC 8484 DNS wireformat codec and HTTP/2 support (httpx) for Quad9/OpenDNS DoH providers. - Quarantine dead, parked, or seized mirrors for the session on hard failure (DNS errors, connection refused, 410/451, parked pages) while preserving bypass clearance on live mirrors. - Add background startup search warmup to prime DNS, elect mirrors, and pre-solve protection challenges to eliminate cold-start search latency. - Add comprehensive test suites for DoH wireformat, mirror quarantine, parked domain detection, and search warmup. --- pyproject.toml | 3 + shelfmark/download/doh_wireformat.py | 158 +++++++++++ shelfmark/download/http.py | 52 +++- shelfmark/download/network.py | 263 +++++++++++++++--- shelfmark/download/warmup.py | 119 ++++++++ shelfmark/main.py | 5 + shelfmark/release_sources/direct_download.py | 83 +++++- .../test_search_parked_mirror.py | 107 +++++++ tests/download/test_doh_wireformat.py | 143 ++++++++++ tests/download/test_http_aa_redirects.py | 7 +- .../test_network_mirror_quarantine.py | 221 +++++++++++++++ tests/download/test_search_warmup.py | 146 ++++++++++ uv.lock | 80 +++++- 13 files changed, 1326 insertions(+), 61 deletions(-) create mode 100644 shelfmark/download/doh_wireformat.py create mode 100644 shelfmark/download/warmup.py create mode 100644 tests/direct_download/test_search_parked_mirror.py create mode 100644 tests/download/test_doh_wireformat.py create mode 100644 tests/download/test_network_mirror_quarantine.py create mode 100644 tests/download/test_search_warmup.py diff --git a/pyproject.toml b/pyproject.toml index 14446e3a..bf0799e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,9 @@ dependencies = [ "transmission-rpc", "authlib>=1.7.2,<1.8", "apprise>=1.12.0", + # HTTP/2 client for RFC 8484 DoH: quad9 rejects HTTP/1.1 outright (505), which + # requests cannot speak. See shelfmark/download/doh_wireformat.py. + "httpx[http2]>=0.27", ] [project.optional-dependencies] diff --git a/shelfmark/download/doh_wireformat.py b/shelfmark/download/doh_wireformat.py new file mode 100644 index 00000000..da2111d8 --- /dev/null +++ b/shelfmark/download/doh_wireformat.py @@ -0,0 +1,158 @@ +"""RFC 8484 DNS wireformat encoding/decoding for DoH providers. + +Providers split into two incompatible camps and the difference is not cosmetic: + +* **JSON** (Cloudflare, Google) - ``?name=&type=A`` returning a JSON body. A + convention, not a standard, and the only one Shelfmark used to speak. +* **Wireformat** (Quad9, OpenDNS) - RFC 8484 proper: a base64url-encoded DNS message + in ``?dns=``, answered with ``application/dns-message``. Quad9 additionally + *requires HTTP/2* per RFC 8484 section 5.2 and answers HTTP/1.1 with 505. + +This module carries the codec only; the transport choice lives in the resolver. +Encoding a query is a handful of bytes, and parsing an answer needs message +compression support (RFC 1035 section 4.1.4) because answer names are almost always +pointers back into the question. +""" + +from __future__ import annotations + +import base64 +import secrets +import struct + +# Record types we resolve. +TYPE_A = 1 +TYPE_AAAA = 28 + +_CLASS_IN = 1 +_HEADER = struct.Struct(">HHHHHH") +_RR_FIXED = struct.Struct(">HHIH") # type, class, ttl, rdlength +_FLAG_RECURSION_DESIRED = 0x0100 +_MAX_LABEL_JUMPS = 64 # cap pointer-following so a malicious answer cannot loop +_MAX_NAME_LENGTH = 255 + + +class WireformatError(ValueError): + """Raised when a DNS wireformat message cannot be parsed.""" + + +def encode_query(hostname: str, record_type: int) -> bytes: + """Build a DNS query message for ``hostname``. + + The ID is zero because RFC 8484 section 4.1 requires it for cacheability, but the + caller may randomise it when not using a cache. + """ + if not hostname: + msg = "hostname must not be empty" + raise WireformatError(msg) + + question = bytearray() + for label in hostname.rstrip(".").split("."): + encoded = label.encode("idna") if not label.isascii() else label.encode("ascii") + if not encoded or len(encoded) > 63: + msg = f"invalid DNS label in {hostname!r}" + raise WireformatError(msg) + question.append(len(encoded)) + question.extend(encoded) + question.append(0) + question.extend(struct.pack(">HH", record_type, _CLASS_IN)) + + header = _HEADER.pack(0, _FLAG_RECURSION_DESIRED, 1, 0, 0, 0) + return header + bytes(question) + + +def encode_query_param(hostname: str, record_type: int) -> str: + """Return the base64url ``dns=`` parameter value for a query (padding stripped).""" + return base64.urlsafe_b64encode(encode_query(hostname, record_type)).rstrip(b"=").decode() + + +def _read_name(message: bytes, offset: int) -> int: + """Skip over a (possibly compressed) name, returning the offset after it.""" + jumps = 0 + length = 0 + while True: + if offset >= len(message): + msg = "truncated DNS name" + raise WireformatError(msg) + label_len = message[offset] + if label_len == 0: + return offset + 1 + if label_len & 0xC0 == 0xC0: + # A pointer ends this name; the rest of the record follows the 2 bytes. + if offset + 1 >= len(message): + msg = "truncated DNS name pointer" + raise WireformatError(msg) + return offset + 2 + offset += 1 + label_len + length += 1 + label_len + jumps += 1 + if jumps > _MAX_LABEL_JUMPS or length > _MAX_NAME_LENGTH: + msg = "malformed DNS name" + raise WireformatError(msg) + + +def decode_answer(message: bytes, record_type: int) -> list[str]: + """Extract the IP addresses of ``record_type`` from a DNS response message. + + Returns an empty list for a well-formed response that carries no matching record + (NXDOMAIN, or only CNAMEs), and raises WireformatError for a malformed one - the + caller treats those differently. + """ + if len(message) < _HEADER.size: + msg = "DNS response shorter than its header" + raise WireformatError(msg) + + _id, _flags, qdcount, ancount, _ns, _ar = _HEADER.unpack_from(message, 0) + offset = _HEADER.size + + for _ in range(qdcount): + offset = _read_name(message, offset) + offset += 4 # QTYPE + QCLASS + + results: list[str] = [] + for _ in range(ancount): + offset = _read_name(message, offset) + if offset + _RR_FIXED.size > len(message): + msg = "truncated resource record" + raise WireformatError(msg) + rtype, rclass, _ttl, rdlength = _RR_FIXED.unpack_from(message, offset) + offset += _RR_FIXED.size + rdata = message[offset : offset + rdlength] + if len(rdata) != rdlength: + msg = "truncated record data" + raise WireformatError(msg) + offset += rdlength + + if rclass != _CLASS_IN or rtype != record_type: + continue + if rtype == TYPE_A and rdlength == 4: + results.append(".".join(str(b) for b in rdata)) + elif rtype == TYPE_AAAA and rdlength == 16: + groups = struct.unpack(">8H", rdata) + results.append(_compress_ipv6(groups)) + + return results + + +def _compress_ipv6(groups: tuple[int, ...]) -> str: + """Render an IPv6 address with the longest zero run collapsed to '::'.""" + best_start = best_len = -1 + run_start = -1 + for i, group in enumerate([*list(groups), 1]): # sentinel closes a trailing run + if group == 0 and i < len(groups): + if run_start < 0: + run_start = i + elif run_start >= 0: + if i - run_start > best_len: + best_start, best_len = run_start, i - run_start + run_start = -1 + + parts = [format(g, "x") for g in groups] + if best_len > 1: + return ":".join(parts[:best_start]) + "::" + ":".join(parts[best_start + best_len :]) + return ":".join(parts) + + +def random_query_id() -> int: + """A random DNS message ID, for callers that do not want the RFC 8484 zero.""" + return secrets.randbelow(0x10000) diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index 241200bd..0ef21cbc 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -234,13 +234,50 @@ def _is_retryable_error(e: Exception) -> bool: return status is not None and status in RETRYABLE_CODES +# Statuses that mean the host is gone rather than busy: 410 Gone and 451 Unavailable +# For Legal Reasons are what a seized domain answers with. +_DEAD_MIRROR_CODES = (410, 451) + + +def _fatal_mirror_reason(e: Exception) -> str | None: + """Return why ``e`` proves the mirror is unusable, or None if it may recover. + + Hard evidence only - the name does not resolve, nothing is listening, or the host + says it is gone for good. A timeout, a 5xx or a challenge all mean the mirror is + alive, and rotating off it discards the bypass clearance held for that domain. + """ + status = _get_status_code(e) + if status is not None and status in _DEAD_MIRROR_CODES: + return f"HTTP {status}" + + # requests wraps the real cause; a read timeout subclasses ConnectionError for + # some adapters, so exclude timeouts explicitly before inspecting the message. + if isinstance(e, requests.exceptions.Timeout): + return None + if not isinstance(e, requests.exceptions.ConnectionError): + return None + + text = str(e).lower() + if "nameresolutionerror" in text or "failed to resolve" in text or "name or service" in text: + return "DNS does not resolve" + if "connection refused" in text or "no route to host" in text: + return "connection refused" + return None + + def _try_rotation( - original_url: str, current_url: str, selector: network.AAMirrorSelector + original_url: str, + current_url: str, + selector: network.AAMirrorSelector, + *, + fatal_reason: str | None = None, ) -> str | None: """Try mirror/DNS rotation. Returns new URL or None.""" aa_base_url = network.get_aa_base_url() if aa_base_url and current_url.startswith(aa_base_url): - new_base, action = selector.next_mirror_or_rotate_dns() + new_base, action = selector.next_mirror_or_rotate_dns( + fatal=fatal_reason is not None, reason=fatal_reason or "" + ) if action in ("mirror", "dns") and new_base: new_url = selector.rewrite(original_url) logger.info("[%s] switching to: %s", action, new_url) @@ -552,9 +589,14 @@ def html_get_page( logger.warning("404 error: %s", current_url) return _result("", current_url) - # Try mirror/DNS rotation on retryable errors - if _is_retryable_error(e): - new_url = _try_rotation(original_url, current_url, selector) + # Try mirror/DNS rotation on retryable errors. A failure that proves the + # mirror is unusable also drops it from this process's rotation, so the + # next search does not pay for it again. + fatal_reason = _fatal_mirror_reason(e) + if fatal_reason or _is_retryable_error(e): + new_url = _try_rotation( + original_url, current_url, selector, fatal_reason=fatal_reason + ) if new_url: current_url = new_url handshake_cookies.clear() diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index 309104a5..ff3e63b4 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -11,6 +11,7 @@ from socket import AddressFamily, SocketKind from typing import TYPE_CHECKING, Any, cast import dns.resolver +import httpx import requests from dns.exception import DNSException @@ -277,6 +278,14 @@ _current_aa_url_index = 0 _aa_urls: list[str] = [] # Initialized lazily in _initialize_aa_state() _aa_base_url: str = "" # Current active AA URL +# Mirrors quarantined for this process: domains that are not a working AA mirror at +# all (NXDOMAIN, refused, or a 200 that isn't AA - seized/parked/for-sale domains all +# land here). Kept separate from ordinary failures: a 403 challenge or a 5xx means the +# mirror is alive and rotating away from it only discards the DDoS-Guard clearance we +# hold for it. Deliberately in-memory only, so a restart re-probes everything. +_dead_aa_urls: set[str] = set() +_dead_aa_urls_lock = _RLock() + def _ensure_initialized() -> None: """Lazy guard so runtime setup happens once and late calls still work.""" @@ -298,6 +307,24 @@ DNS_PROVIDERS = [ ("opendns", ["208.67.222.222", "208.67.220.220"], "https://doh.opendns.com/dns-query"), ] +# httpx raises its own hierarchy, which shares no base class with requests', so a +# wireformat failure would escape a requests-only except clause. +_DOH_REQUEST_ERRORS = (OSError, ValueError, requests.RequestException, httpx.HTTPError) + + +def _first_proxy(proxies: dict[str, str] | None) -> str | None: + """Pick a single proxy URL from a requests-style mapping, for httpx.""" + if not proxies: + return None + return proxies.get("https") or proxies.get("http") or None + + +# DoH providers that speak RFC 8484 wireformat rather than the (non-standard) JSON API +# Cloudflare and Google popularised. Verified against the live services: both reject a +# ?name=&type= query outright - Quad9 with 505 (it also mandates HTTP/2 per RFC 8484 +# section 5.2, which requests cannot speak), OpenDNS with 400 "No valid query received". +_DOH_WIREFORMAT_HOSTS = frozenset({"dns.quad9.net", "doh.opendns.com"}) + # Domain patterns that should trigger DNS rotation on failure DNS_ROTATION_DOMAINS = [ "annas-archive", @@ -462,8 +489,16 @@ class DoHResolver: # DNS cache: {(hostname, record_type): (ip_list, timestamp)} self._cache: dict[tuple[str, str], tuple[list[str], datetime]] = {} - # Different headers based on provider - if "google" in self.base_url: + # RFC 8484 providers get a separate transport: they need wireformat, and Quad9 + # additionally refuses HTTP/1.1, which requests has no way to upgrade from. + self.use_wireformat = urllib.parse.urlparse(self.base_url).hostname in ( + _DOH_WIREFORMAT_HOSTS + ) + self._http2_client: Any | None = None + + if self.use_wireformat: + self.session.headers.update({"Accept": "application/dns-message"}) + elif "google" in self.base_url: self.session.headers.update( { "Accept": "application/json", @@ -476,6 +511,35 @@ class DoHResolver: } ) + def _get_http2_client(self) -> Any: + """Lazily build the HTTP/2 client used for RFC 8484 providers. + + Built on first use so a resolver pointed at a JSON provider never opens an + HTTP/2 connection pool it will not use. + """ + if self._http2_client is None: + self._http2_client = httpx.Client( + http2=True, + timeout=10, + verify=get_ssl_verify(self.base_url), + proxy=_first_proxy(get_proxies(self.base_url)), + ) + return self._http2_client + + def _resolve_wireformat(self, hostname: str, record_type: str) -> list[str]: + """Resolve via RFC 8484: base64url query in, DNS message out.""" + from shelfmark.download import doh_wireformat + + qtype = doh_wireformat.TYPE_AAAA if record_type == "AAAA" else doh_wireformat.TYPE_A + param = doh_wireformat.encode_query_param(hostname, qtype) + response = self._get_http2_client().get( + self.base_url, + params={"dns": param}, + headers={"Accept": "application/dns-message"}, + ) + response.raise_for_status() + return doh_wireformat.decode_answer(response.content, qtype) + def _get_cached(self, hostname: str, record_type: str) -> list[str] | None: """Get cached DNS result if still valid.""" key = (hostname, record_type) @@ -525,34 +589,37 @@ class DoHResolver: return cached try: - params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"} + if self.use_wireformat: + answers = self._resolve_wireformat(hostname, record_type) + else: + params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"} - response = self.session.get( - self.base_url, - params=params, - proxies=get_proxies(self.base_url), - timeout=10, # Increased from 5s to handle slow network conditions - verify=get_ssl_verify(self.base_url), - ) - response.raise_for_status() + response = self.session.get( + self.base_url, + params=params, + proxies=get_proxies(self.base_url), + timeout=10, # Increased from 5s to handle slow network conditions + verify=get_ssl_verify(self.base_url), + ) + response.raise_for_status() - data = response.json() - if "Answer" not in data: - logger.warning("DoH resolution failed for %s: %s", hostname, data) - return [] + data = response.json() + if "Answer" not in data: + logger.warning("DoH resolution failed for %s: %s", hostname, data) + return [] - # Extract IP addresses from the response - answers = [ - answer["data"] - for answer in data["Answer"] - if answer.get("type") == (28 if record_type == "AAAA" else 1) - ] + # Extract IP addresses from the response + answers = [ + answer["data"] + for answer in data["Answer"] + if answer.get("type") == (28 if record_type == "AAAA" else 1) + ] # Cache the result self._set_cached(hostname, record_type, answers) # Don't log here - the caller (custom_getaddrinfo) will log the final result - except (OSError, ValueError, requests.RequestException) as e: + except _DOH_REQUEST_ERRORS as e: logger.warning("DoH resolution failed for %s: %s", hostname, e) return [] else: @@ -1015,14 +1082,18 @@ def rotate_dns_and_reset_aa() -> bool: configured_url = _get_configured_aa_url() if configured_url == "auto": - # Auto mode always resets to the first mirror to restart the cascade - _current_aa_url_index = 0 - if _aa_urls: - _aa_base_url = _aa_urls[0] + # Auto mode always resets to the first mirror to restart the cascade. Skip any + # quarantined ones: a new DNS provider cannot revive a parked or seized domain. + with _dead_aa_urls_lock: + restart_urls = [url for url in _aa_urls if url not in _dead_aa_urls] or _aa_urls + if restart_urls: + _aa_base_url = restart_urls[0] + _current_aa_url_index = _aa_urls.index(_aa_base_url) logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url) _save_state(aa_url=_aa_base_url) else: _aa_base_url = "" + _current_aa_url_index = 0 logger.info("After DNS switch, AA URL remains unconfigured") else: # Keep the user's configured primary mirror (if it exists in the list), @@ -1192,8 +1263,17 @@ def _initialize_aa_state() -> None: global _aa_base_url, _current_aa_url_index, _aa_urls # Build URL list from config + previous_urls = _aa_urls _aa_urls = _build_aa_urls() + # Drop quarantine decisions only when the mirror list itself changed - they were + # made about a list that no longer applies. This runs on every re-init (settings + # sync, DNS rotation, helper subprocess startup), and clearing unconditionally + # would resurrect a parked mirror mid-session. + if previous_urls != _aa_urls: + with _dead_aa_urls_lock: + _dead_aa_urls.clear() + # Get configured base URL from config configured_url = _get_configured_aa_url() @@ -1209,26 +1289,34 @@ def _initialize_aa_state() -> None: return if configured_url == "auto": - if state.get("aa_base_url") and state["aa_base_url"] in _aa_urls: - _current_aa_url_index = _aa_urls.index(state["aa_base_url"]) - _aa_base_url = state["aa_base_url"] + # Never restore or probe a mirror quarantined this session: re-init happens + # often, and re-electing a parked domain costs a wasted request every time + # (its parking page answers 200, so the probe would happily pick it). + with _dead_aa_urls_lock: + candidates = [url for url in _aa_urls if url not in _dead_aa_urls] + restored = state.get("aa_base_url") + if restored and restored in candidates: + _current_aa_url_index = _aa_urls.index(restored) + _aa_base_url = restored else: - logger.debug("AA_BASE_URL: auto, checking available urls %s", _aa_urls) - for i, url in enumerate(_aa_urls): + logger.debug("AA_BASE_URL: auto, checking available urls %s", candidates) + for url in candidates: try: response = requests.get( url, proxies=get_proxies(url), timeout=3, verify=get_ssl_verify(url) ) if response.status_code == HTTPStatus.OK: - _current_aa_url_index = i + _current_aa_url_index = _aa_urls.index(url) _aa_base_url = url _save_state(aa_url=_aa_base_url) break except (OSError, requests.RequestException) as exc: logger.debug("Could not reach AA mirror candidate %s: %s", url, exc) - if not _aa_base_url or _aa_base_url == "auto": - _aa_base_url = _aa_urls[0] - _current_aa_url_index = 0 + # Also covers the case where every probe failed and the previous base is + # itself quarantined - keeping it would aim the next search at a dead host. + if not _aa_base_url or _aa_base_url == "auto" or _aa_base_url not in candidates: + _aa_base_url = (candidates or _aa_urls)[0] + _current_aa_url_index = _aa_urls.index(_aa_base_url) elif configured_url not in _aa_urls: logger.info("AA_BASE_URL set to custom value %s; skipping auto-switch", configured_url) _aa_base_url = configured_url @@ -1326,24 +1414,77 @@ def is_aa_auto_mode() -> bool: def get_available_aa_urls() -> list[str]: - """Get list of configured AA URLs (copy).""" + """Get configured AA URLs (copy), minus any quarantined this process. + + Falls back to the full list when every mirror has been quarantined: a wrong + classification must not leave the app with nowhere to search. + """ _ensure_initialized() - return _aa_urls.copy() + with _dead_aa_urls_lock: + alive = [url for url in _aa_urls if url not in _dead_aa_urls] + if not alive and _aa_urls: + logger.warning("All AA mirrors quarantined; retrying the full list") + _dead_aa_urls.clear() + return _aa_urls.copy() + return alive -def set_aa_url_index(new_index: int) -> bool: - """Set AA base URL by index in available list; returns True if applied.""" +def _aa_base_for_url(url: str) -> str: + """Return the configured mirror base that ``url`` belongs to, if any.""" + for base in _aa_urls: + if base and url.startswith(base): + return base + return "" + + +def mark_aa_url_dead(url: str, reason: str) -> bool: + """Quarantine an AA mirror for the rest of this process. + + Only for hard evidence that the host is not a working AA mirror. Transient + failures (403 challenge, 429, 5xx, timeouts) must never come through here - + quarantining a live mirror throws away its bypass clearance. + """ + _ensure_initialized() + base = _aa_base_for_url(url) or url + with _dead_aa_urls_lock: + if base not in _aa_urls or base in _dead_aa_urls: + return False + # Keep at least one mirror in play, even if it is the failing one. + if len([u for u in _aa_urls if u not in _dead_aa_urls]) <= 1: + logger.warning("Not quarantining last remaining AA mirror %s (%s)", base, reason) + return False + _dead_aa_urls.add(base) + logger.warning("Quarantined AA mirror %s for this session: %s", base, reason) + return True + + +def get_dead_aa_urls() -> set[str]: + """Return the mirrors quarantined this process (copy).""" + with _dead_aa_urls_lock: + return set(_dead_aa_urls) + + +def set_aa_url(url: str) -> bool: + """Set the active AA base URL; returns True if applied.""" _ensure_initialized() global _aa_base_url, _current_aa_url_index - if new_index < 0 or new_index >= len(_aa_urls): + if url not in _aa_urls: return False - _current_aa_url_index = new_index - _aa_base_url = _aa_urls[_current_aa_url_index] + _current_aa_url_index = _aa_urls.index(url) + _aa_base_url = url logger.info("Set AA URL to: %s", _aa_base_url) _save_state(aa_url=_aa_base_url) return True +def set_aa_url_index(new_index: int) -> bool: + """Set AA base URL by index in the full configured list; True if applied.""" + _ensure_initialized() + if new_index < 0 or new_index >= len(_aa_urls): + return False + return set_aa_url(_aa_urls[new_index]) + + class AAMirrorSelector: """Keep AA mirror switching consistent across call sites. @@ -1357,6 +1498,10 @@ class AAMirrorSelector: def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None: _ensure_initialized() self.aa_urls = get_available_aa_urls() + # Rotation walks the live mirrors, but rewriting has to recognise every + # configured base: a URL built before a mirror was quarantined still points at + # it, and failing to rewrite would send the retry back to the dead host. + self.all_aa_urls = _aa_urls.copy() self._index = self._safe_index(get_aa_base_url()) self.current_base = self.aa_urls[self._index] if self.aa_urls else "" if reset_attempts: @@ -1369,16 +1514,41 @@ class AAMirrorSelector: def rewrite(self, url: str) -> str: """Replace any known AA base in url with current_base.""" - for base in self.aa_urls: + for base in self.all_aa_urls: if url.startswith(base): return url.replace(base, self.current_base, 1) return url - def next_mirror_or_rotate_dns(self, *, allow_dns: bool = True) -> tuple[str | None, str]: + def quarantine_current(self, reason: str) -> bool: + """Quarantine the mirror this selector is on (hard failures only).""" + if not self.current_base: + return False + dropped = mark_aa_url_dead(self.current_base, reason) + if dropped: + # Rebuild from the surviving mirrors so the dead one is out of the cycle. + self._ensure_fresh_state(reset_attempts=False) + return dropped + + def next_mirror_or_rotate_dns( + self, *, allow_dns: bool = True, fatal: bool = False, reason: str = "" + ) -> tuple[str | None, str]: """Advance to the next mirror or rotate DNS if needed. + ``fatal`` marks the current mirror as not-an-AA-mirror (NXDOMAIN, refused, a + 200 that isn't AA) and drops it from this process's rotation. Leave it False + for anything the mirror can recover from - a challenge or a 5xx means the host + is alive, and quarantining it would discard its bypass clearance. + Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'. """ + if fatal and self.quarantine_current(reason or "unusable mirror"): + # Quarantining rebuilt the state onto a surviving mirror, so that mirror is + # the next one to try - advancing again here would skip straight past it. + self.attempts_this_dns += 1 + if self.current_base and is_aa_auto_mode(): + set_aa_url(self.current_base) + return self.current_base, "mirror" + self.attempts_this_dns += 1 max_attempts = len(self.aa_urls) if is_aa_auto_mode() else 1 if self.attempts_this_dns >= max_attempts: @@ -1391,8 +1561,11 @@ class AAMirrorSelector: # Mirror is explicitly configured; do not fail over to other mirrors. return None, "exhausted" + if not self.aa_urls: + return None, "exhausted" + next_index = (self._index + 1) % len(self.aa_urls) - set_aa_url_index(next_index) + set_aa_url(self.aa_urls[next_index]) self._ensure_fresh_state(reset_attempts=False) return self.current_base, "mirror" diff --git a/shelfmark/download/warmup.py b/shelfmark/download/warmup.py new file mode 100644 index 00000000..67e565c6 --- /dev/null +++ b/shelfmark/download/warmup.py @@ -0,0 +1,119 @@ +"""Boot-time warm-up of the direct-download source. + +The first AA search after a cold start pays for the whole cold path at once: DNS +resolution, electing a live mirror, spinning up headless Chrome and solving the +DDoS-Guard challenge. That is tens of seconds with the user sat at the search box. + +Running one throwaway search shortly after boot moves that cost off the user's first +search. It primes the DNS cache, elects (and quarantines) mirrors, and leaves the +clearance cookie in the bypasser's per-domain cache, so the first real search reuses +it instead of solving from scratch. + +Runs on a daemon thread and swallows every failure: this is an optimisation, and a +source that is down at boot must not affect startup or health. +""" + +from __future__ import annotations + +import threading + +from shelfmark.core.config import config +from shelfmark.core.logger import setup_logger + +logger = setup_logger(__name__) + +# Delay before the warm-up fires. Long enough that it does not compete with the rest +# of startup (and with a container's own health probe) for the first request. +_DEFAULT_DELAY_SECONDS = 15.0 + +_DEFAULT_QUERY = "The Great Gatsby" + +_warmup_thread: threading.Thread | None = None +_warmup_lock = threading.Lock() + + +def _as_bool(value: object, *, default: bool) -> bool: + """Coerce a config value that may arrive as a string, bool or None.""" + if value is None: + return default + if isinstance(value, str): + from shelfmark.config.env import string_to_bool + + return string_to_bool(value) + return bool(value) + + +def is_enabled() -> bool: + """Whether the boot-time warm-up search should run.""" + if not _as_bool(config.get("SEARCH_WARMUP_ENABLED", True), default=True): + return False + # Nothing to warm if the source is off, and no challenge to pre-solve without + # the bypasser - a plain search is fast enough not to need this. + if not _as_bool(config.get("DIRECT_DOWNLOAD_ENABLED", True), default=True): + logger.debug("Search warm-up skipped: direct download disabled") + return False + return True + + +def warmup_query() -> str: + """The query used to warm the source.""" + raw = config.get("SEARCH_WARMUP_QUERY", _DEFAULT_QUERY) + query = str(raw).strip() if raw else "" + return query or _DEFAULT_QUERY + + +def run_warmup() -> bool: + """Run one warm-up search. Returns True if it produced results. + + Never raises: every failure mode here is one the next real search would hit + anyway, and reporting it is the search path's job, not the warm-up's. + """ + from shelfmark.core.mirrors import has_aa_mirror_configuration + + if not has_aa_mirror_configuration(): + logger.debug("Search warm-up skipped: no Anna's Archive mirrors configured") + return False + + query = warmup_query() + logger.info("Warming up direct download search (%r)", query) + try: + from shelfmark.core.models import SearchFilters + from shelfmark.release_sources.direct_download import search_books + + results = search_books(query, SearchFilters()) + except Exception: + # Broad by design: a warm-up must never take the app down, and the source + # raises everything from network errors to parse failures. + logger.warning("Search warm-up did not complete; first user search may be slow") + logger.debug("Search warm-up failure detail", exc_info=True) + return False + + if results: + logger.info("Search warm-up complete: %s results, source is ready", len(results)) + return True + logger.info("Search warm-up returned no results; source reachable but empty") + return False + + +def start(delay_seconds: float = _DEFAULT_DELAY_SECONDS) -> bool: + """Schedule the warm-up on a daemon thread. Safe to call multiple times.""" + global _warmup_thread + + if not is_enabled(): + return False + + with _warmup_lock: + if _warmup_thread is not None and _warmup_thread.is_alive(): + logger.debug("Search warm-up already scheduled") + return False + + def _run() -> None: + run_warmup() + + _warmup_thread = threading.Timer(delay_seconds, _run) + _warmup_thread.daemon = True + _warmup_thread.name = "SearchWarmup" + _warmup_thread.start() + + logger.debug("Search warm-up scheduled in %ss", delay_seconds) + return True diff --git a/shelfmark/main.py b/shelfmark/main.py index 78157dd1..04f1f736 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -85,6 +85,7 @@ from shelfmark.core.requests_service import ( from shelfmark.core.user_db import UserDB from shelfmark.core.utils import AUDIOBOOK_FORMATS, normalize_base_path from shelfmark.download import orchestrator as backend +from shelfmark.download import warmup from shelfmark.release_sources import ( BrowseRecord, Release, @@ -206,6 +207,10 @@ except (sqlite3.OperationalError, OSError) as e: # Start download coordinator backend.start() +# Pre-solve the direct-download source's protection challenge in the background so the +# first user search does not pay for a cold Chrome bypass. Never blocks startup. +warmup.start() + # Rate limiting for login attempts # Map usernames to their failed-attempt counters and lockout timestamps. failed_login_attempts: dict[str, dict[str, Any]] = {} diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index 68ad07df..281586c1 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -539,6 +539,79 @@ class SearchUnavailableError(SourceUnavailableError): """Raised when Anna's Archive cannot be reached via any mirror/DNS.""" +# Markers that prove a 200 really came from Anna's Archive, and markers that mean we +# are looking at a protection interstitial rather than the site. A page with neither +# is a domain that answers but is not AA - seized, parked or for sale. +# +# Deliberately structural rather than the domain name: a parking page's whole job is +# to display the domain it is squatting on, so "annas-archive" matches the very pages +# this is meant to catch. These paths only exist on the real site. +_AA_PAGE_MARKERS = ( + "/md5/", + "aarecord", + "anna's archive", + "/dyn/", + "/datasets", + "/fast_download", + "/slow_download", +) +_CHALLENGE_MARKERS = ( + "ddos-guard", + "just a moment", + "cloudflare", + "checking your browser", + "cf-browser-verification", +) + + +def _looks_like_aa_page(html: str) -> bool: + """Whether ``html`` is recognisably Anna's Archive, or a challenge in front of it.""" + lowered = html.lower() + return any(marker in lowered for marker in (*_AA_PAGE_MARKERS, *_CHALLENGE_MARKERS)) + + +def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[str, Tag | None]: + """Fetch the AA search page, retrying past mirrors that are not actually AA. + + A parked or seized domain answers 200 with a page that has no results table and no + "No files found." - indistinguishable from a broken search unless we check whether + the response looks like AA at all. Those mirrors are quarantined for the session so + later searches skip them instead of paying the timeout again. + """ + attempt_url = url + for _ in range(len(network.get_available_aa_urls()) or 1): + response = downloader.html_get_page( + attempt_url, selector=selector, allow_bypasser_fallback=True + ) + if not response: + # Network/mirror exhaustion path bubbles up so API can notify clients + msg = "Unable to reach download source. Network restricted or mirrors are blocked." + raise SearchUnavailableError(msg) + + html = _html_response_text(response) + soup = BeautifulSoup(html, "html.parser") + table = soup.find("table") + if isinstance(table, Tag): + return html, table + if table is not None: + msg = f"Expected results table tag, got {type(table).__name__}" + raise TypeError(msg) + if "No files found." in html or _looks_like_aa_page(html): + # A real AA response - either genuinely empty, or a shape the caller + # should report as drift. Not the mirror's fault. + return html, None + + new_base, action = selector.next_mirror_or_rotate_dns( + fatal=True, reason="responded without an Anna's Archive page" + ) + if action not in ("mirror", "dns") or not new_base: + return html, None + attempt_url = selector.rewrite(url) + logger.info("Retrying search on %s", new_base) + + return "", None + + def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: """Search for books matching the query. @@ -603,15 +676,7 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: # AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating # to another mirror only collects another 403, so let the bypasser solve it. - html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True) - if not html: - # Network/mirror exhaustion path bubbles up so API can notify clients - msg = "Unable to reach download source. Network restricted or mirrors are blocked." - raise SearchUnavailableError(msg) - - soup = BeautifulSoup(_html_response_text(html), "html.parser") - tbody = soup.find("table") - + html, tbody = _fetch_search_table(url, selector) if tbody is None: if "No files found." in html: logger.info("No books found for query: %s", query) diff --git a/tests/direct_download/test_search_parked_mirror.py b/tests/direct_download/test_search_parked_mirror.py new file mode 100644 index 00000000..d245644a --- /dev/null +++ b/tests/direct_download/test_search_parked_mirror.py @@ -0,0 +1,107 @@ +"""A mirror that answers 200 with a non-AA page is quarantined, not reported as empty. + +Seized and for-sale domains keep serving 200. Without a look at *what* came back, a +parking page is indistinguishable from a broken search, so the mirror stays in +rotation and every later search pays for it again. +""" + +from bs4 import Tag + +PARKED_PAGE = """annas-archive.li +

This domain is for sale

Inquire now. Buy this domain.

""" + +AA_RESULTS_PAGE = """
+ +
Dune
""" + +AA_EMPTY_PAGE = """
+
No files found.
about +
""" + +DDOS_GUARD_PAGE = """DDoS-Guard +
Checking your browser
""" + + +class _Selector: + def __init__(self, bases: list[str]) -> None: + self._bases = bases + self._index = 0 + self.current_base = bases[0] + self.quarantined: list[str] = [] + + def rewrite(self, url: str) -> str: + for base in self._bases: + if url.startswith(base): + return url.replace(base, self.current_base, 1) + return url + + def next_mirror_or_rotate_dns(self, *, fatal: bool = False, reason: str = ""): + if fatal: + self.quarantined.append(self.current_base) + self._index += 1 + if self._index >= len(self._bases): + return None, "exhausted" + self.current_base = self._bases[self._index] + return self.current_base, "mirror" + + +def _patch_pages(monkeypatch, pages: list[str]): + """Serve `pages` in order, recording the URL each call was made against.""" + import shelfmark.release_sources.direct_download as dd + + calls: list[str] = [] + + def fake_get(url, **_kwargs): + calls.append(url) + return pages[len(calls) - 1] if len(calls) <= len(pages) else "" + + monkeypatch.setattr(dd.downloader, "html_get_page", fake_get) + monkeypatch.setattr(dd.network, "get_available_aa_urls", lambda: ["a", "b", "c"]) + return dd, calls + + +def test_parked_mirror_is_quarantined_and_search_retries_next_mirror(monkeypatch): + dd, calls = _patch_pages(monkeypatch, [PARKED_PAGE, AA_RESULTS_PAGE]) + selector = _Selector(["https://parked.test", "https://real.test"]) + + html, table = dd._fetch_search_table("https://parked.test/search?q=dune", selector) + + assert selector.quarantined == ["https://parked.test"] + assert isinstance(table, Tag) + assert "Dune" in html + # The retry went to the live mirror, not back to the parked one. + assert calls[1].startswith("https://real.test") + + +def test_genuinely_empty_aa_result_does_not_quarantine(monkeypatch): + """'No files found.' is a real answer from a healthy mirror.""" + dd, _calls = _patch_pages(monkeypatch, [AA_EMPTY_PAGE]) + selector = _Selector(["https://real.test", "https://other.test"]) + + html, table = dd._fetch_search_table("https://real.test/search?q=zzz", selector) + + assert selector.quarantined == [] + assert table is None + assert "No files found." in html + + +def test_challenge_page_does_not_quarantine(monkeypatch): + """A DDoS-Guard interstitial means the mirror is alive and holds our clearance.""" + dd, _calls = _patch_pages(monkeypatch, [DDOS_GUARD_PAGE]) + selector = _Selector(["https://real.test", "https://other.test"]) + + _html, table = dd._fetch_search_table("https://real.test/search?q=dune", selector) + + assert selector.quarantined == [] + assert table is None + + +def test_unreachable_mirror_raises_search_unavailable(monkeypatch): + dd, _calls = _patch_pages(monkeypatch, [""]) + selector = _Selector(["https://real.test"]) + + try: + dd._fetch_search_table("https://real.test/search?q=dune", selector) + except dd.SearchUnavailableError: + return + raise AssertionError("expected SearchUnavailableError") diff --git a/tests/download/test_doh_wireformat.py b/tests/download/test_doh_wireformat.py new file mode 100644 index 00000000..e4aad4f9 --- /dev/null +++ b/tests/download/test_doh_wireformat.py @@ -0,0 +1,143 @@ +"""RFC 8484 wireformat codec. + +Quad9 and OpenDNS reject the JSON API that Cloudflare and Google popularised, so +these providers only work through wireformat. Quad9 additionally requires HTTP/2 +(section 5.2) and answers HTTP/1.1 with 505. +""" + +import base64 +import struct + +import pytest + +from shelfmark.download import doh_wireformat as wf + + +def _decode_param(param: str) -> bytes: + padding = "=" * (-len(param) % 4) + return base64.urlsafe_b64decode(param + padding) + + +def _build_response( + *, qname: str = "example.com", answers: list[tuple[int, bytes]], qtype: int = wf.TYPE_A +) -> bytes: + """Assemble a response whose answer names are compression pointers to the question.""" + question = b"" + for label in qname.split("."): + question += bytes([len(label)]) + label.encode() + question += b"\x00" + struct.pack(">HH", qtype, 1) + + body = b"" + for rtype, rdata in answers: + body += b"\xc0\x0c" # pointer to offset 12 (the question name) + body += struct.pack(">HHIH", rtype, 1, 300, len(rdata)) + rdata + + header = struct.pack(">HHHHHH", 0, 0x8180, 1, len(answers), 0, 0) + return header + question + body + + +def test_encode_query_is_a_well_formed_dns_message(): + raw = _decode_param(wf.encode_query_param("example.com", wf.TYPE_A)) + + msg_id, flags, qdcount, ancount, _ns, _ar = struct.unpack_from(">HHHHHH", raw, 0) + assert msg_id == 0 # RFC 8484 section 4.1: zero for cacheability + assert flags == 0x0100 # recursion desired + assert (qdcount, ancount) == (1, 0) + assert raw[12:] == b"\x07example\x03com\x00" + struct.pack(">HH", wf.TYPE_A, 1) + + +def test_encode_query_param_is_unpadded_base64url(): + param = wf.encode_query_param("example.com", wf.TYPE_A) + assert "=" not in param + assert "+" not in param and "/" not in param + + +def test_encode_query_strips_trailing_dot(): + assert _decode_param(wf.encode_query_param("example.com.", wf.TYPE_A)) == _decode_param( + wf.encode_query_param("example.com", wf.TYPE_A) + ) + + +def test_encode_query_rejects_empty_hostname(): + with pytest.raises(wf.WireformatError): + wf.encode_query("", wf.TYPE_A) + + +def test_encode_query_rejects_oversized_label(): + with pytest.raises(wf.WireformatError): + wf.encode_query("a" * 64 + ".com", wf.TYPE_A) + + +def test_decode_a_records(): + response = _build_response(answers=[(wf.TYPE_A, bytes([93, 184, 216, 34]))]) + assert wf.decode_answer(response, wf.TYPE_A) == ["93.184.216.34"] + + +def test_decode_multiple_a_records_preserves_order(): + response = _build_response( + answers=[(wf.TYPE_A, bytes([1, 1, 1, 1])), (wf.TYPE_A, bytes([8, 8, 8, 8]))] + ) + assert wf.decode_answer(response, wf.TYPE_A) == ["1.1.1.1", "8.8.8.8"] + + +def test_decode_skips_cname_records_in_the_chain(): + """Answers routinely lead with a CNAME; only the requested type is an address.""" + cname = b"\x03www\x07example\x03com\x00" + response = _build_response(answers=[(5, cname), (wf.TYPE_A, bytes([93, 184, 216, 34]))]) + assert wf.decode_answer(response, wf.TYPE_A) == ["93.184.216.34"] + + +def test_decode_aaaa_compresses_zero_run(): + # 2606:4700:0:0:0:0:6810:84e5 -> the middle zero run collapses to "::" + rdata = struct.pack(">8H", 0x2606, 0x4700, 0, 0, 0, 0, 0x6810, 0x84E5) + response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA) + assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2606:4700::6810:84e5"] + + +def test_decode_aaaa_collapses_only_the_longest_zero_run(): + # 2001:0:0:1:0:0:0:1 - the second, longer run is the one that collapses. + rdata = struct.pack(">8H", 0x2001, 0, 0, 1, 0, 0, 0, 1) + response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA) + assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2001:0:0:1::1"] + + +def test_decode_aaaa_without_zero_run(): + rdata = struct.pack(">8H", 0x2001, 0x0DB8, 1, 2, 3, 4, 5, 6) + response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA) + assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2001:db8:1:2:3:4:5:6"] + + +def test_decode_nxdomain_returns_empty_not_an_error(): + """An empty answer is a valid response; the caller falls back rather than retrying.""" + response = _build_response(answers=[]) + assert wf.decode_answer(response, wf.TYPE_A) == [] + + +def test_decode_rejects_truncated_header(): + with pytest.raises(wf.WireformatError): + wf.decode_answer(b"\x00\x01", wf.TYPE_A) + + +def test_decode_rejects_truncated_record(): + response = _build_response(answers=[(wf.TYPE_A, bytes([1, 2, 3, 4]))]) + with pytest.raises(wf.WireformatError): + wf.decode_answer(response[:-2], wf.TYPE_A) + + +def test_decode_does_not_hang_on_a_malicious_name(): + """A self-referential name must not loop forever.""" + header = struct.pack(">HHHHHH", 0, 0x8180, 1, 0, 0, 0) + # A run of maximum-length labels that never terminates. + body = (b"\x3f" + b"a" * 63) * 8 + with pytest.raises(wf.WireformatError): + wf.decode_answer(header + body, wf.TYPE_A) + + +def test_wireformat_providers_are_flagged(monkeypatch): + """The provider table and the resolver must agree on who needs wireformat.""" + import shelfmark.download.network as network + + for name, servers, url in network.DNS_PROVIDERS: + resolver = network.DoHResolver(url, "x.invalid", servers[0]) + expected = name in ("quad9", "opendns") + assert resolver.use_wireformat is expected, f"{name} wireformat flag wrong" diff --git a/tests/download/test_http_aa_redirects.py b/tests/download/test_http_aa_redirects.py index 87ccaa2d..ee7df692 100644 --- a/tests/download/test_http_aa_redirects.py +++ b/tests/download/test_http_aa_redirects.py @@ -27,6 +27,7 @@ class _DummySelector: self._index = 0 self.current_base = bases[0] self.attempts_this_dns = 0 + self.quarantined: list[tuple[str, str]] = [] def rewrite(self, url: str) -> str: for base in self._bases: @@ -34,7 +35,11 @@ class _DummySelector: return url.replace(base, self.current_base, 1) return url - def next_mirror_or_rotate_dns(self, allow_dns: bool = True) -> tuple[str | None, str]: + def next_mirror_or_rotate_dns( + self, allow_dns: bool = True, *, fatal: bool = False, reason: str = "" + ) -> tuple[str | None, str]: + if fatal: + self.quarantined.append((self.current_base, reason)) self.attempts_this_dns += 1 self._index = (self._index + 1) % len(self._bases) self.current_base = self._bases[self._index] diff --git a/tests/download/test_network_mirror_quarantine.py b/tests/download/test_network_mirror_quarantine.py new file mode 100644 index 00000000..7206817c --- /dev/null +++ b/tests/download/test_network_mirror_quarantine.py @@ -0,0 +1,221 @@ +"""Tests for AA mirror quarantine: dead mirrors leave the rotation, live ones stay. + +The distinction these guard is the whole point of the feature. A mirror that answers +403 (DDoS-Guard) is alive and holds our bypass clearance, so rotating off it makes the +next search solve a fresh challenge on a domain we have no cookie for. A mirror that +NXDOMAINs, refuses the connection, or answers 200 with a parking page is not a mirror +at all and must never be tried again this session. +""" + +import requests + + +def _fresh_network(monkeypatch, urls: list[str], *, auto: bool = True): + import shelfmark.download.network as network + + monkeypatch.setattr(network, "_initialized", True) + monkeypatch.setattr(network, "_aa_urls", list(urls)) + monkeypatch.setattr(network, "_aa_base_url", urls[0]) + monkeypatch.setattr(network, "_current_aa_url_index", 0) + monkeypatch.setattr(network, "_dead_aa_urls", set()) + monkeypatch.setattr(network, "_save_state", lambda **kwargs: None) + monkeypatch.setattr(network, "is_aa_auto_mode", lambda: auto) + return network + + +MIRRORS = ["https://aa-one.test", "https://aa-two.test", "https://aa-three.test"] + + +def test_quarantined_mirror_leaves_the_available_list(monkeypatch): + network = _fresh_network(monkeypatch, MIRRORS) + + assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True + assert network.get_available_aa_urls() == ["https://aa-one.test", "https://aa-three.test"] + assert network.get_dead_aa_urls() == {"https://aa-two.test"} + + +def test_quarantine_accepts_a_full_request_url(monkeypatch): + """Callers hold the failing request URL, not the bare mirror base.""" + network = _fresh_network(monkeypatch, MIRRORS) + + assert network.mark_aa_url_dead("https://aa-two.test/search?q=dune", "parked") is True + assert "https://aa-two.test" in network.get_dead_aa_urls() + + +def test_quarantine_is_idempotent(monkeypatch): + network = _fresh_network(monkeypatch, MIRRORS) + + assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True + assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is False + assert network.get_available_aa_urls() == ["https://aa-one.test", "https://aa-three.test"] + + +def test_last_surviving_mirror_is_never_quarantined(monkeypatch): + """Misclassification must not leave the app with nowhere to search.""" + network = _fresh_network(monkeypatch, MIRRORS) + + assert network.mark_aa_url_dead("https://aa-one.test", "NXDOMAIN") is True + assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True + assert network.mark_aa_url_dead("https://aa-three.test", "NXDOMAIN") is False + assert network.get_available_aa_urls() == ["https://aa-three.test"] + + +def test_selector_skips_quarantined_mirror_when_rotating(monkeypatch): + network = _fresh_network(monkeypatch, MIRRORS) + selector = network.AAMirrorSelector() + + new_base, action = selector.next_mirror_or_rotate_dns(fatal=True, reason="NXDOMAIN") + + assert action == "mirror" + # Landed on the next live mirror, not skipped past it onto the third. + assert new_base == "https://aa-two.test" + assert "https://aa-one.test" in network.get_dead_aa_urls() + assert selector.rewrite("https://aa-one.test/search") == "https://aa-two.test/search" + + +def test_non_fatal_rotation_keeps_the_mirror(monkeypatch): + """A 5xx or a challenge rotates but must not burn the mirror.""" + network = _fresh_network(monkeypatch, MIRRORS) + selector = network.AAMirrorSelector() + + selector.next_mirror_or_rotate_dns() + + assert network.get_dead_aa_urls() == set() + assert network.get_available_aa_urls() == MIRRORS + + +def test_dns_reset_does_not_resurrect_quarantined_mirrors(monkeypatch): + """A new DNS provider cannot revive a parked domain, so it stays skipped.""" + network = _fresh_network(monkeypatch, MIRRORS) + monkeypatch.setattr(network, "rotate_dns_provider", lambda: True) + monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto") + network.mark_aa_url_dead("https://aa-one.test", "parked") + + assert network.rotate_dns_and_reset_aa() is True + assert network.get_aa_base_url() == "https://aa-two.test" + + +def test_editing_the_mirror_list_clears_quarantine(monkeypatch): + """Quarantine decisions were made about a list the user has now changed.""" + network = _fresh_network(monkeypatch, MIRRORS) + network.mark_aa_url_dead("https://aa-two.test", "parked") + monkeypatch.setattr(network, "_build_aa_urls", lambda: [*MIRRORS, "https://aa-four.test"]) + monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto") + monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"}) + + network._initialize_aa_state() + + assert network.get_dead_aa_urls() == set() + + +def test_reinit_with_an_unchanged_list_keeps_quarantine(monkeypatch): + """Re-init happens constantly (settings sync, DNS rotation, helper startup). + + Clearing quarantine on every one of those resurrects a parked mirror mid-session, + which is exactly the bug this guards: the mirror gets re-elected and the next + search pays for it again. + """ + network = _fresh_network(monkeypatch, MIRRORS) + network.mark_aa_url_dead("https://aa-two.test", "parked") + monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS)) + monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto") + monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"}) + + network._initialize_aa_state() + + assert network.get_dead_aa_urls() == {"https://aa-two.test"} + + +# --------------------------------------------------------------------------- # +# Failure classification +# --------------------------------------------------------------------------- # +def _http_error(status: int) -> requests.exceptions.HTTPError: + response = requests.Response() + response.status_code = status + return requests.exceptions.HTTPError(response=response) + + +def test_dns_failure_is_fatal_for_the_mirror(): + import shelfmark.download.http as http + + exc = requests.exceptions.ConnectionError( + "HTTPSConnectionPool(host='aa.test', port=443): Max retries exceeded " + "(Caused by NameResolutionError(\"Failed to resolve 'aa.test'\"))" + ) + assert http._fatal_mirror_reason(exc) == "DNS does not resolve" + + +def test_connection_refused_is_fatal_for_the_mirror(): + import shelfmark.download.http as http + + exc = requests.exceptions.ConnectionError("Connection refused") + assert http._fatal_mirror_reason(exc) == "connection refused" + + +def test_gone_and_legal_block_are_fatal(): + import shelfmark.download.http as http + + assert http._fatal_mirror_reason(_http_error(410)) == "HTTP 410" + assert http._fatal_mirror_reason(_http_error(451)) == "HTTP 451" + + +def test_timeout_is_not_fatal(): + """A slow mirror is still a mirror - and may hold our bypass clearance.""" + import shelfmark.download.http as http + + assert http._fatal_mirror_reason(requests.exceptions.ConnectTimeout("timed out")) is None + assert http._fatal_mirror_reason(requests.exceptions.ReadTimeout("timed out")) is None + + +def test_challenge_and_server_errors_are_not_fatal(): + import shelfmark.download.http as http + + for status in (403, 429, 500, 502, 503): + assert http._fatal_mirror_reason(_http_error(status)) is None + + +def test_startup_probe_skips_quarantined_mirrors(monkeypatch): + """Re-init must not re-probe (or re-elect) a mirror already known to be dead. + + A parking page answers 200, so an unfiltered probe elects it every single time + the app re-initialises - one wasted request per re-init, forever. + """ + import requests + + network = _fresh_network(monkeypatch, MIRRORS) + network.mark_aa_url_dead("https://aa-one.test", "parked") + probed: list[str] = [] + + def fake_get(url, **_kwargs): + probed.append(url) + response = requests.Response() + response.status_code = 200 + return response + + monkeypatch.setattr(network.requests, "get", fake_get) + monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS)) + monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto") + monkeypatch.setattr(network, "state", {}) + monkeypatch.setattr(network, "get_proxies", lambda _url: None) + monkeypatch.setattr(network, "get_ssl_verify", lambda _url: True) + + network._initialize_aa_state() + + assert "https://aa-one.test" not in probed + assert network.get_aa_base_url() == "https://aa-two.test" + + +def test_startup_probe_does_not_restore_a_quarantined_mirror(monkeypatch): + """Saved state can name a mirror that has since been quarantined.""" + network = _fresh_network(monkeypatch, MIRRORS) + network.mark_aa_url_dead("https://aa-one.test", "parked") + monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS)) + monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto") + monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"}) + monkeypatch.setattr(network, "get_proxies", lambda _url: None) + monkeypatch.setattr(network, "get_ssl_verify", lambda _url: True) + monkeypatch.setattr(network.requests, "get", lambda *a, **k: (_ for _ in ()).throw(OSError())) + + network._initialize_aa_state() + + assert network.get_aa_base_url() != "https://aa-one.test" diff --git a/tests/download/test_search_warmup.py b/tests/download/test_search_warmup.py new file mode 100644 index 00000000..9bbccf16 --- /dev/null +++ b/tests/download/test_search_warmup.py @@ -0,0 +1,146 @@ +"""Boot-time search warm-up. + +The warm-up exists to move the cold DDoS-Guard solve off the user's first search. It +is an optimisation, so the load-bearing property is that it can never affect startup: +a source that is down, misconfigured or raising must leave the app running. +""" + +import pytest + + +@pytest.fixture +def warmup(monkeypatch): + import shelfmark.download.warmup as warmup_module + + monkeypatch.setattr(warmup_module, "_warmup_thread", None) + return warmup_module + + +def _patch_config(monkeypatch, warmup, values: dict): + def fake_get(key, default=None): + return values.get(key, default) + + monkeypatch.setattr(warmup.config, "get", fake_get) + + +def test_disabled_by_setting(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": False}) + assert warmup.is_enabled() is False + assert warmup.start() is False + + +def test_disabled_by_string_false(monkeypatch, warmup): + """Deployment ENV arrives as a string, not a bool.""" + _patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": "false"}) + assert warmup.is_enabled() is False + + +def test_skipped_when_direct_download_is_off(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {"DIRECT_DOWNLOAD_ENABLED": False}) + assert warmup.is_enabled() is False + + +def test_enabled_by_default(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {}) + assert warmup.is_enabled() is True + + +def test_query_defaults_and_is_configurable(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {}) + assert warmup.warmup_query() == "The Great Gatsby" + + _patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "Dune"}) + assert warmup.warmup_query() == "Dune" + + # A blank override must not send an empty query at the source. + _patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": " "}) + assert warmup.warmup_query() == "The Great Gatsby" + + +def test_skipped_when_no_mirrors_configured(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {}) + import shelfmark.core.mirrors as mirrors + + monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: False) + + called: list[str] = [] + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(dd, "search_books", lambda q, f: called.append(q)) + + assert warmup.run_warmup() is False + assert called == [] + + +def test_successful_warmup_reports_true(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {}) + import shelfmark.core.mirrors as mirrors + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True) + seen: list[str] = [] + + def fake_search(query, _filters): + seen.append(query) + return ["a", "b"] + + monkeypatch.setattr(dd, "search_books", fake_search) + + assert warmup.run_warmup() is True + assert seen == ["The Great Gatsby"] + + +def test_empty_results_are_not_an_error(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {}) + import shelfmark.core.mirrors as mirrors + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True) + monkeypatch.setattr(dd, "search_books", lambda q, f: []) + + assert warmup.run_warmup() is False + + +def test_search_failure_is_swallowed(monkeypatch, warmup): + """A source that is down at boot must not propagate out of the warm-up.""" + _patch_config(monkeypatch, warmup, {}) + import shelfmark.core.mirrors as mirrors + import shelfmark.release_sources.direct_download as dd + + monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True) + + def boom(_query, _filters): + msg = "mirrors are blocked" + raise RuntimeError(msg) + + monkeypatch.setattr(dd, "search_books", boom) + + assert warmup.run_warmup() is False + + +def test_start_schedules_a_daemon_thread_and_is_idempotent(monkeypatch, warmup): + _patch_config(monkeypatch, warmup, {}) + + assert warmup.start(delay_seconds=30) is True + thread = warmup._warmup_thread + assert thread is not None + assert thread.daemon is True + + # A second call must not stack up another timer. + assert warmup.start(delay_seconds=30) is False + assert warmup._warmup_thread is thread + + thread.cancel() + + +def test_start_does_not_run_the_search_inline(monkeypatch, warmup): + """Startup must not block on a search that can take a minute.""" + _patch_config(monkeypatch, warmup, {}) + ran: list[bool] = [] + monkeypatch.setattr(warmup, "run_warmup", lambda: ran.append(True)) + + warmup.start(delay_seconds=30) + assert ran == [] + + if warmup._warmup_thread: + warmup._warmup_thread.cancel() diff --git a/uv.lock b/uv.lock index 0653203e..f78e6f5d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,7 +1,19 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.14" +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + [[package]] name = "apprise" version = "1.12.0" @@ -546,6 +558,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -1391,6 +1467,7 @@ dependencies = [ { name = "gevent" }, { name = "gevent-websocket" }, { name = "gunicorn" }, + { name = "httpx", extra = ["http2"] }, { name = "psutil" }, { name = "python-socketio" }, { name = "qbittorrent-api" }, @@ -1433,6 +1510,7 @@ requires-dist = [ { name = "gevent" }, { name = "gevent-websocket" }, { name = "gunicorn" }, + { name = "httpx", extras = ["http2"], specifier = ">=0.27" }, { name = "psutil" }, { name = "pyautogui", marker = "extra == 'browser'" }, { name = "python-socketio" },