mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-10-04 22:05:45 +01:00
feat(download): add DoH wireformat support, mirror quarantine, and search warmup (#1218)
- Add RFC 8484 DNS wireformat codec and HTTP/2 support (httpx) for Quad9/OpenDNS DoH providers. - Quarantine dead, parked, or seized mirrors for the session on hard failure (DNS errors, connection refused, 410/451, parked pages) while preserving bypass clearance on live mirrors. - Add background startup search warmup to prime DNS, elect mirrors, and pre-solve protection challenges to eliminate cold-start search latency. - Add comprehensive test suites for DoH wireformat, mirror quarantine, parked domain detection, and search warmup.
This commit is contained in:
CaliBrain
@@ -23,6 +23,9 @@ dependencies = [
|
||||
"transmission-rpc",
|
||||
"authlib>=1.7.2,<1.8",
|
||||
"apprise>=1.12.0",
|
||||
# HTTP/2 client for RFC 8484 DoH: quad9 rejects HTTP/1.1 outright (505), which
|
||||
# requests cannot speak. See shelfmark/download/doh_wireformat.py.
|
||||
"httpx[http2]>=0.27",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""RFC 8484 DNS wireformat encoding/decoding for DoH providers.
|
||||
|
||||
Providers split into two incompatible camps and the difference is not cosmetic:
|
||||
|
||||
* **JSON** (Cloudflare, Google) - ``?name=<host>&type=A`` returning a JSON body. A
|
||||
convention, not a standard, and the only one Shelfmark used to speak.
|
||||
* **Wireformat** (Quad9, OpenDNS) - RFC 8484 proper: a base64url-encoded DNS message
|
||||
in ``?dns=``, answered with ``application/dns-message``. Quad9 additionally
|
||||
*requires HTTP/2* per RFC 8484 section 5.2 and answers HTTP/1.1 with 505.
|
||||
|
||||
This module carries the codec only; the transport choice lives in the resolver.
|
||||
Encoding a query is a handful of bytes, and parsing an answer needs message
|
||||
compression support (RFC 1035 section 4.1.4) because answer names are almost always
|
||||
pointers back into the question.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import secrets
|
||||
import struct
|
||||
|
||||
# Record types we resolve.
|
||||
TYPE_A = 1
|
||||
TYPE_AAAA = 28
|
||||
|
||||
_CLASS_IN = 1
|
||||
_HEADER = struct.Struct(">HHHHHH")
|
||||
_RR_FIXED = struct.Struct(">HHIH") # type, class, ttl, rdlength
|
||||
_FLAG_RECURSION_DESIRED = 0x0100
|
||||
_MAX_LABEL_JUMPS = 64 # cap pointer-following so a malicious answer cannot loop
|
||||
_MAX_NAME_LENGTH = 255
|
||||
|
||||
|
||||
class WireformatError(ValueError):
|
||||
"""Raised when a DNS wireformat message cannot be parsed."""
|
||||
|
||||
|
||||
def encode_query(hostname: str, record_type: int) -> bytes:
|
||||
"""Build a DNS query message for ``hostname``.
|
||||
|
||||
The ID is zero because RFC 8484 section 4.1 requires it for cacheability, but the
|
||||
caller may randomise it when not using a cache.
|
||||
"""
|
||||
if not hostname:
|
||||
msg = "hostname must not be empty"
|
||||
raise WireformatError(msg)
|
||||
|
||||
question = bytearray()
|
||||
for label in hostname.rstrip(".").split("."):
|
||||
encoded = label.encode("idna") if not label.isascii() else label.encode("ascii")
|
||||
if not encoded or len(encoded) > 63:
|
||||
msg = f"invalid DNS label in {hostname!r}"
|
||||
raise WireformatError(msg)
|
||||
question.append(len(encoded))
|
||||
question.extend(encoded)
|
||||
question.append(0)
|
||||
question.extend(struct.pack(">HH", record_type, _CLASS_IN))
|
||||
|
||||
header = _HEADER.pack(0, _FLAG_RECURSION_DESIRED, 1, 0, 0, 0)
|
||||
return header + bytes(question)
|
||||
|
||||
|
||||
def encode_query_param(hostname: str, record_type: int) -> str:
|
||||
"""Return the base64url ``dns=`` parameter value for a query (padding stripped)."""
|
||||
return base64.urlsafe_b64encode(encode_query(hostname, record_type)).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def _read_name(message: bytes, offset: int) -> int:
|
||||
"""Skip over a (possibly compressed) name, returning the offset after it."""
|
||||
jumps = 0
|
||||
length = 0
|
||||
while True:
|
||||
if offset >= len(message):
|
||||
msg = "truncated DNS name"
|
||||
raise WireformatError(msg)
|
||||
label_len = message[offset]
|
||||
if label_len == 0:
|
||||
return offset + 1
|
||||
if label_len & 0xC0 == 0xC0:
|
||||
# A pointer ends this name; the rest of the record follows the 2 bytes.
|
||||
if offset + 1 >= len(message):
|
||||
msg = "truncated DNS name pointer"
|
||||
raise WireformatError(msg)
|
||||
return offset + 2
|
||||
offset += 1 + label_len
|
||||
length += 1 + label_len
|
||||
jumps += 1
|
||||
if jumps > _MAX_LABEL_JUMPS or length > _MAX_NAME_LENGTH:
|
||||
msg = "malformed DNS name"
|
||||
raise WireformatError(msg)
|
||||
|
||||
|
||||
def decode_answer(message: bytes, record_type: int) -> list[str]:
|
||||
"""Extract the IP addresses of ``record_type`` from a DNS response message.
|
||||
|
||||
Returns an empty list for a well-formed response that carries no matching record
|
||||
(NXDOMAIN, or only CNAMEs), and raises WireformatError for a malformed one - the
|
||||
caller treats those differently.
|
||||
"""
|
||||
if len(message) < _HEADER.size:
|
||||
msg = "DNS response shorter than its header"
|
||||
raise WireformatError(msg)
|
||||
|
||||
_id, _flags, qdcount, ancount, _ns, _ar = _HEADER.unpack_from(message, 0)
|
||||
offset = _HEADER.size
|
||||
|
||||
for _ in range(qdcount):
|
||||
offset = _read_name(message, offset)
|
||||
offset += 4 # QTYPE + QCLASS
|
||||
|
||||
results: list[str] = []
|
||||
for _ in range(ancount):
|
||||
offset = _read_name(message, offset)
|
||||
if offset + _RR_FIXED.size > len(message):
|
||||
msg = "truncated resource record"
|
||||
raise WireformatError(msg)
|
||||
rtype, rclass, _ttl, rdlength = _RR_FIXED.unpack_from(message, offset)
|
||||
offset += _RR_FIXED.size
|
||||
rdata = message[offset : offset + rdlength]
|
||||
if len(rdata) != rdlength:
|
||||
msg = "truncated record data"
|
||||
raise WireformatError(msg)
|
||||
offset += rdlength
|
||||
|
||||
if rclass != _CLASS_IN or rtype != record_type:
|
||||
continue
|
||||
if rtype == TYPE_A and rdlength == 4:
|
||||
results.append(".".join(str(b) for b in rdata))
|
||||
elif rtype == TYPE_AAAA and rdlength == 16:
|
||||
groups = struct.unpack(">8H", rdata)
|
||||
results.append(_compress_ipv6(groups))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _compress_ipv6(groups: tuple[int, ...]) -> str:
|
||||
"""Render an IPv6 address with the longest zero run collapsed to '::'."""
|
||||
best_start = best_len = -1
|
||||
run_start = -1
|
||||
for i, group in enumerate([*list(groups), 1]): # sentinel closes a trailing run
|
||||
if group == 0 and i < len(groups):
|
||||
if run_start < 0:
|
||||
run_start = i
|
||||
elif run_start >= 0:
|
||||
if i - run_start > best_len:
|
||||
best_start, best_len = run_start, i - run_start
|
||||
run_start = -1
|
||||
|
||||
parts = [format(g, "x") for g in groups]
|
||||
if best_len > 1:
|
||||
return ":".join(parts[:best_start]) + "::" + ":".join(parts[best_start + best_len :])
|
||||
return ":".join(parts)
|
||||
|
||||
|
||||
def random_query_id() -> int:
|
||||
"""A random DNS message ID, for callers that do not want the RFC 8484 zero."""
|
||||
return secrets.randbelow(0x10000)
|
||||
@@ -234,13 +234,50 @@ def _is_retryable_error(e: Exception) -> bool:
|
||||
return status is not None and status in RETRYABLE_CODES
|
||||
|
||||
|
||||
# Statuses that mean the host is gone rather than busy: 410 Gone and 451 Unavailable
|
||||
# For Legal Reasons are what a seized domain answers with.
|
||||
_DEAD_MIRROR_CODES = (410, 451)
|
||||
|
||||
|
||||
def _fatal_mirror_reason(e: Exception) -> str | None:
|
||||
"""Return why ``e`` proves the mirror is unusable, or None if it may recover.
|
||||
|
||||
Hard evidence only - the name does not resolve, nothing is listening, or the host
|
||||
says it is gone for good. A timeout, a 5xx or a challenge all mean the mirror is
|
||||
alive, and rotating off it discards the bypass clearance held for that domain.
|
||||
"""
|
||||
status = _get_status_code(e)
|
||||
if status is not None and status in _DEAD_MIRROR_CODES:
|
||||
return f"HTTP {status}"
|
||||
|
||||
# requests wraps the real cause; a read timeout subclasses ConnectionError for
|
||||
# some adapters, so exclude timeouts explicitly before inspecting the message.
|
||||
if isinstance(e, requests.exceptions.Timeout):
|
||||
return None
|
||||
if not isinstance(e, requests.exceptions.ConnectionError):
|
||||
return None
|
||||
|
||||
text = str(e).lower()
|
||||
if "nameresolutionerror" in text or "failed to resolve" in text or "name or service" in text:
|
||||
return "DNS does not resolve"
|
||||
if "connection refused" in text or "no route to host" in text:
|
||||
return "connection refused"
|
||||
return None
|
||||
|
||||
|
||||
def _try_rotation(
|
||||
original_url: str, current_url: str, selector: network.AAMirrorSelector
|
||||
original_url: str,
|
||||
current_url: str,
|
||||
selector: network.AAMirrorSelector,
|
||||
*,
|
||||
fatal_reason: str | None = None,
|
||||
) -> str | None:
|
||||
"""Try mirror/DNS rotation. Returns new URL or None."""
|
||||
aa_base_url = network.get_aa_base_url()
|
||||
if aa_base_url and current_url.startswith(aa_base_url):
|
||||
new_base, action = selector.next_mirror_or_rotate_dns()
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=fatal_reason is not None, reason=fatal_reason or ""
|
||||
)
|
||||
if action in ("mirror", "dns") and new_base:
|
||||
new_url = selector.rewrite(original_url)
|
||||
logger.info("[%s] switching to: %s", action, new_url)
|
||||
@@ -552,9 +589,14 @@ def html_get_page(
|
||||
logger.warning("404 error: %s", current_url)
|
||||
return _result("", current_url)
|
||||
|
||||
# Try mirror/DNS rotation on retryable errors
|
||||
if _is_retryable_error(e):
|
||||
new_url = _try_rotation(original_url, current_url, selector)
|
||||
# Try mirror/DNS rotation on retryable errors. A failure that proves the
|
||||
# mirror is unusable also drops it from this process's rotation, so the
|
||||
# next search does not pay for it again.
|
||||
fatal_reason = _fatal_mirror_reason(e)
|
||||
if fatal_reason or _is_retryable_error(e):
|
||||
new_url = _try_rotation(
|
||||
original_url, current_url, selector, fatal_reason=fatal_reason
|
||||
)
|
||||
if new_url:
|
||||
current_url = new_url
|
||||
handshake_cookies.clear()
|
||||
|
||||
+218
-45
@@ -11,6 +11,7 @@ from socket import AddressFamily, SocketKind
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import dns.resolver
|
||||
import httpx
|
||||
import requests
|
||||
from dns.exception import DNSException
|
||||
|
||||
@@ -277,6 +278,14 @@ _current_aa_url_index = 0
|
||||
_aa_urls: list[str] = [] # Initialized lazily in _initialize_aa_state()
|
||||
_aa_base_url: str = "" # Current active AA URL
|
||||
|
||||
# Mirrors quarantined for this process: domains that are not a working AA mirror at
|
||||
# all (NXDOMAIN, refused, or a 200 that isn't AA - seized/parked/for-sale domains all
|
||||
# land here). Kept separate from ordinary failures: a 403 challenge or a 5xx means the
|
||||
# mirror is alive and rotating away from it only discards the DDoS-Guard clearance we
|
||||
# hold for it. Deliberately in-memory only, so a restart re-probes everything.
|
||||
_dead_aa_urls: set[str] = set()
|
||||
_dead_aa_urls_lock = _RLock()
|
||||
|
||||
|
||||
def _ensure_initialized() -> None:
|
||||
"""Lazy guard so runtime setup happens once and late calls still work."""
|
||||
@@ -298,6 +307,24 @@ DNS_PROVIDERS = [
|
||||
("opendns", ["208.67.222.222", "208.67.220.220"], "https://doh.opendns.com/dns-query"),
|
||||
]
|
||||
|
||||
# httpx raises its own hierarchy, which shares no base class with requests', so a
|
||||
# wireformat failure would escape a requests-only except clause.
|
||||
_DOH_REQUEST_ERRORS = (OSError, ValueError, requests.RequestException, httpx.HTTPError)
|
||||
|
||||
|
||||
def _first_proxy(proxies: dict[str, str] | None) -> str | None:
|
||||
"""Pick a single proxy URL from a requests-style mapping, for httpx."""
|
||||
if not proxies:
|
||||
return None
|
||||
return proxies.get("https") or proxies.get("http") or None
|
||||
|
||||
|
||||
# DoH providers that speak RFC 8484 wireformat rather than the (non-standard) JSON API
|
||||
# Cloudflare and Google popularised. Verified against the live services: both reject a
|
||||
# ?name=&type= query outright - Quad9 with 505 (it also mandates HTTP/2 per RFC 8484
|
||||
# section 5.2, which requests cannot speak), OpenDNS with 400 "No valid query received".
|
||||
_DOH_WIREFORMAT_HOSTS = frozenset({"dns.quad9.net", "doh.opendns.com"})
|
||||
|
||||
# Domain patterns that should trigger DNS rotation on failure
|
||||
DNS_ROTATION_DOMAINS = [
|
||||
"annas-archive",
|
||||
@@ -462,8 +489,16 @@ class DoHResolver:
|
||||
# DNS cache: {(hostname, record_type): (ip_list, timestamp)}
|
||||
self._cache: dict[tuple[str, str], tuple[list[str], datetime]] = {}
|
||||
|
||||
# Different headers based on provider
|
||||
if "google" in self.base_url:
|
||||
# RFC 8484 providers get a separate transport: they need wireformat, and Quad9
|
||||
# additionally refuses HTTP/1.1, which requests has no way to upgrade from.
|
||||
self.use_wireformat = urllib.parse.urlparse(self.base_url).hostname in (
|
||||
_DOH_WIREFORMAT_HOSTS
|
||||
)
|
||||
self._http2_client: Any | None = None
|
||||
|
||||
if self.use_wireformat:
|
||||
self.session.headers.update({"Accept": "application/dns-message"})
|
||||
elif "google" in self.base_url:
|
||||
self.session.headers.update(
|
||||
{
|
||||
"Accept": "application/json",
|
||||
@@ -476,6 +511,35 @@ class DoHResolver:
|
||||
}
|
||||
)
|
||||
|
||||
def _get_http2_client(self) -> Any:
|
||||
"""Lazily build the HTTP/2 client used for RFC 8484 providers.
|
||||
|
||||
Built on first use so a resolver pointed at a JSON provider never opens an
|
||||
HTTP/2 connection pool it will not use.
|
||||
"""
|
||||
if self._http2_client is None:
|
||||
self._http2_client = httpx.Client(
|
||||
http2=True,
|
||||
timeout=10,
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
proxy=_first_proxy(get_proxies(self.base_url)),
|
||||
)
|
||||
return self._http2_client
|
||||
|
||||
def _resolve_wireformat(self, hostname: str, record_type: str) -> list[str]:
|
||||
"""Resolve via RFC 8484: base64url query in, DNS message out."""
|
||||
from shelfmark.download import doh_wireformat
|
||||
|
||||
qtype = doh_wireformat.TYPE_AAAA if record_type == "AAAA" else doh_wireformat.TYPE_A
|
||||
param = doh_wireformat.encode_query_param(hostname, qtype)
|
||||
response = self._get_http2_client().get(
|
||||
self.base_url,
|
||||
params={"dns": param},
|
||||
headers={"Accept": "application/dns-message"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return doh_wireformat.decode_answer(response.content, qtype)
|
||||
|
||||
def _get_cached(self, hostname: str, record_type: str) -> list[str] | None:
|
||||
"""Get cached DNS result if still valid."""
|
||||
key = (hostname, record_type)
|
||||
@@ -525,34 +589,37 @@ class DoHResolver:
|
||||
return cached
|
||||
|
||||
try:
|
||||
params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"}
|
||||
if self.use_wireformat:
|
||||
answers = self._resolve_wireformat(hostname, record_type)
|
||||
else:
|
||||
params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"}
|
||||
|
||||
response = self.session.get(
|
||||
self.base_url,
|
||||
params=params,
|
||||
proxies=get_proxies(self.base_url),
|
||||
timeout=10, # Increased from 5s to handle slow network conditions
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
)
|
||||
response.raise_for_status()
|
||||
response = self.session.get(
|
||||
self.base_url,
|
||||
params=params,
|
||||
proxies=get_proxies(self.base_url),
|
||||
timeout=10, # Increased from 5s to handle slow network conditions
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
if "Answer" not in data:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, data)
|
||||
return []
|
||||
data = response.json()
|
||||
if "Answer" not in data:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, data)
|
||||
return []
|
||||
|
||||
# Extract IP addresses from the response
|
||||
answers = [
|
||||
answer["data"]
|
||||
for answer in data["Answer"]
|
||||
if answer.get("type") == (28 if record_type == "AAAA" else 1)
|
||||
]
|
||||
# Extract IP addresses from the response
|
||||
answers = [
|
||||
answer["data"]
|
||||
for answer in data["Answer"]
|
||||
if answer.get("type") == (28 if record_type == "AAAA" else 1)
|
||||
]
|
||||
|
||||
# Cache the result
|
||||
self._set_cached(hostname, record_type, answers)
|
||||
|
||||
# Don't log here - the caller (custom_getaddrinfo) will log the final result
|
||||
except (OSError, ValueError, requests.RequestException) as e:
|
||||
except _DOH_REQUEST_ERRORS as e:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, e)
|
||||
return []
|
||||
else:
|
||||
@@ -1015,14 +1082,18 @@ def rotate_dns_and_reset_aa() -> bool:
|
||||
configured_url = _get_configured_aa_url()
|
||||
|
||||
if configured_url == "auto":
|
||||
# Auto mode always resets to the first mirror to restart the cascade
|
||||
_current_aa_url_index = 0
|
||||
if _aa_urls:
|
||||
_aa_base_url = _aa_urls[0]
|
||||
# Auto mode always resets to the first mirror to restart the cascade. Skip any
|
||||
# quarantined ones: a new DNS provider cannot revive a parked or seized domain.
|
||||
with _dead_aa_urls_lock:
|
||||
restart_urls = [url for url in _aa_urls if url not in _dead_aa_urls] or _aa_urls
|
||||
if restart_urls:
|
||||
_aa_base_url = restart_urls[0]
|
||||
_current_aa_url_index = _aa_urls.index(_aa_base_url)
|
||||
logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url)
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
else:
|
||||
_aa_base_url = ""
|
||||
_current_aa_url_index = 0
|
||||
logger.info("After DNS switch, AA URL remains unconfigured")
|
||||
else:
|
||||
# Keep the user's configured primary mirror (if it exists in the list),
|
||||
@@ -1192,8 +1263,17 @@ def _initialize_aa_state() -> None:
|
||||
global _aa_base_url, _current_aa_url_index, _aa_urls
|
||||
|
||||
# Build URL list from config
|
||||
previous_urls = _aa_urls
|
||||
_aa_urls = _build_aa_urls()
|
||||
|
||||
# Drop quarantine decisions only when the mirror list itself changed - they were
|
||||
# made about a list that no longer applies. This runs on every re-init (settings
|
||||
# sync, DNS rotation, helper subprocess startup), and clearing unconditionally
|
||||
# would resurrect a parked mirror mid-session.
|
||||
if previous_urls != _aa_urls:
|
||||
with _dead_aa_urls_lock:
|
||||
_dead_aa_urls.clear()
|
||||
|
||||
# Get configured base URL from config
|
||||
configured_url = _get_configured_aa_url()
|
||||
|
||||
@@ -1209,26 +1289,34 @@ def _initialize_aa_state() -> None:
|
||||
return
|
||||
|
||||
if configured_url == "auto":
|
||||
if state.get("aa_base_url") and state["aa_base_url"] in _aa_urls:
|
||||
_current_aa_url_index = _aa_urls.index(state["aa_base_url"])
|
||||
_aa_base_url = state["aa_base_url"]
|
||||
# Never restore or probe a mirror quarantined this session: re-init happens
|
||||
# often, and re-electing a parked domain costs a wasted request every time
|
||||
# (its parking page answers 200, so the probe would happily pick it).
|
||||
with _dead_aa_urls_lock:
|
||||
candidates = [url for url in _aa_urls if url not in _dead_aa_urls]
|
||||
restored = state.get("aa_base_url")
|
||||
if restored and restored in candidates:
|
||||
_current_aa_url_index = _aa_urls.index(restored)
|
||||
_aa_base_url = restored
|
||||
else:
|
||||
logger.debug("AA_BASE_URL: auto, checking available urls %s", _aa_urls)
|
||||
for i, url in enumerate(_aa_urls):
|
||||
logger.debug("AA_BASE_URL: auto, checking available urls %s", candidates)
|
||||
for url in candidates:
|
||||
try:
|
||||
response = requests.get(
|
||||
url, proxies=get_proxies(url), timeout=3, verify=get_ssl_verify(url)
|
||||
)
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
_current_aa_url_index = i
|
||||
_current_aa_url_index = _aa_urls.index(url)
|
||||
_aa_base_url = url
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
break
|
||||
except (OSError, requests.RequestException) as exc:
|
||||
logger.debug("Could not reach AA mirror candidate %s: %s", url, exc)
|
||||
if not _aa_base_url or _aa_base_url == "auto":
|
||||
_aa_base_url = _aa_urls[0]
|
||||
_current_aa_url_index = 0
|
||||
# Also covers the case where every probe failed and the previous base is
|
||||
# itself quarantined - keeping it would aim the next search at a dead host.
|
||||
if not _aa_base_url or _aa_base_url == "auto" or _aa_base_url not in candidates:
|
||||
_aa_base_url = (candidates or _aa_urls)[0]
|
||||
_current_aa_url_index = _aa_urls.index(_aa_base_url)
|
||||
elif configured_url not in _aa_urls:
|
||||
logger.info("AA_BASE_URL set to custom value %s; skipping auto-switch", configured_url)
|
||||
_aa_base_url = configured_url
|
||||
@@ -1326,24 +1414,77 @@ def is_aa_auto_mode() -> bool:
|
||||
|
||||
|
||||
def get_available_aa_urls() -> list[str]:
|
||||
"""Get list of configured AA URLs (copy)."""
|
||||
"""Get configured AA URLs (copy), minus any quarantined this process.
|
||||
|
||||
Falls back to the full list when every mirror has been quarantined: a wrong
|
||||
classification must not leave the app with nowhere to search.
|
||||
"""
|
||||
_ensure_initialized()
|
||||
return _aa_urls.copy()
|
||||
with _dead_aa_urls_lock:
|
||||
alive = [url for url in _aa_urls if url not in _dead_aa_urls]
|
||||
if not alive and _aa_urls:
|
||||
logger.warning("All AA mirrors quarantined; retrying the full list")
|
||||
_dead_aa_urls.clear()
|
||||
return _aa_urls.copy()
|
||||
return alive
|
||||
|
||||
|
||||
def set_aa_url_index(new_index: int) -> bool:
|
||||
"""Set AA base URL by index in available list; returns True if applied."""
|
||||
def _aa_base_for_url(url: str) -> str:
|
||||
"""Return the configured mirror base that ``url`` belongs to, if any."""
|
||||
for base in _aa_urls:
|
||||
if base and url.startswith(base):
|
||||
return base
|
||||
return ""
|
||||
|
||||
|
||||
def mark_aa_url_dead(url: str, reason: str) -> bool:
|
||||
"""Quarantine an AA mirror for the rest of this process.
|
||||
|
||||
Only for hard evidence that the host is not a working AA mirror. Transient
|
||||
failures (403 challenge, 429, 5xx, timeouts) must never come through here -
|
||||
quarantining a live mirror throws away its bypass clearance.
|
||||
"""
|
||||
_ensure_initialized()
|
||||
base = _aa_base_for_url(url) or url
|
||||
with _dead_aa_urls_lock:
|
||||
if base not in _aa_urls or base in _dead_aa_urls:
|
||||
return False
|
||||
# Keep at least one mirror in play, even if it is the failing one.
|
||||
if len([u for u in _aa_urls if u not in _dead_aa_urls]) <= 1:
|
||||
logger.warning("Not quarantining last remaining AA mirror %s (%s)", base, reason)
|
||||
return False
|
||||
_dead_aa_urls.add(base)
|
||||
logger.warning("Quarantined AA mirror %s for this session: %s", base, reason)
|
||||
return True
|
||||
|
||||
|
||||
def get_dead_aa_urls() -> set[str]:
|
||||
"""Return the mirrors quarantined this process (copy)."""
|
||||
with _dead_aa_urls_lock:
|
||||
return set(_dead_aa_urls)
|
||||
|
||||
|
||||
def set_aa_url(url: str) -> bool:
|
||||
"""Set the active AA base URL; returns True if applied."""
|
||||
_ensure_initialized()
|
||||
global _aa_base_url, _current_aa_url_index
|
||||
if new_index < 0 or new_index >= len(_aa_urls):
|
||||
if url not in _aa_urls:
|
||||
return False
|
||||
_current_aa_url_index = new_index
|
||||
_aa_base_url = _aa_urls[_current_aa_url_index]
|
||||
_current_aa_url_index = _aa_urls.index(url)
|
||||
_aa_base_url = url
|
||||
logger.info("Set AA URL to: %s", _aa_base_url)
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
return True
|
||||
|
||||
|
||||
def set_aa_url_index(new_index: int) -> bool:
|
||||
"""Set AA base URL by index in the full configured list; True if applied."""
|
||||
_ensure_initialized()
|
||||
if new_index < 0 or new_index >= len(_aa_urls):
|
||||
return False
|
||||
return set_aa_url(_aa_urls[new_index])
|
||||
|
||||
|
||||
class AAMirrorSelector:
|
||||
"""Keep AA mirror switching consistent across call sites.
|
||||
|
||||
@@ -1357,6 +1498,10 @@ class AAMirrorSelector:
|
||||
def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None:
|
||||
_ensure_initialized()
|
||||
self.aa_urls = get_available_aa_urls()
|
||||
# Rotation walks the live mirrors, but rewriting has to recognise every
|
||||
# configured base: a URL built before a mirror was quarantined still points at
|
||||
# it, and failing to rewrite would send the retry back to the dead host.
|
||||
self.all_aa_urls = _aa_urls.copy()
|
||||
self._index = self._safe_index(get_aa_base_url())
|
||||
self.current_base = self.aa_urls[self._index] if self.aa_urls else ""
|
||||
if reset_attempts:
|
||||
@@ -1369,16 +1514,41 @@ class AAMirrorSelector:
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
"""Replace any known AA base in url with current_base."""
|
||||
for base in self.aa_urls:
|
||||
for base in self.all_aa_urls:
|
||||
if url.startswith(base):
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, *, allow_dns: bool = True) -> tuple[str | None, str]:
|
||||
def quarantine_current(self, reason: str) -> bool:
|
||||
"""Quarantine the mirror this selector is on (hard failures only)."""
|
||||
if not self.current_base:
|
||||
return False
|
||||
dropped = mark_aa_url_dead(self.current_base, reason)
|
||||
if dropped:
|
||||
# Rebuild from the surviving mirrors so the dead one is out of the cycle.
|
||||
self._ensure_fresh_state(reset_attempts=False)
|
||||
return dropped
|
||||
|
||||
def next_mirror_or_rotate_dns(
|
||||
self, *, allow_dns: bool = True, fatal: bool = False, reason: str = ""
|
||||
) -> tuple[str | None, str]:
|
||||
"""Advance to the next mirror or rotate DNS if needed.
|
||||
|
||||
``fatal`` marks the current mirror as not-an-AA-mirror (NXDOMAIN, refused, a
|
||||
200 that isn't AA) and drops it from this process's rotation. Leave it False
|
||||
for anything the mirror can recover from - a challenge or a 5xx means the host
|
||||
is alive, and quarantining it would discard its bypass clearance.
|
||||
|
||||
Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'.
|
||||
"""
|
||||
if fatal and self.quarantine_current(reason or "unusable mirror"):
|
||||
# Quarantining rebuilt the state onto a surviving mirror, so that mirror is
|
||||
# the next one to try - advancing again here would skip straight past it.
|
||||
self.attempts_this_dns += 1
|
||||
if self.current_base and is_aa_auto_mode():
|
||||
set_aa_url(self.current_base)
|
||||
return self.current_base, "mirror"
|
||||
|
||||
self.attempts_this_dns += 1
|
||||
max_attempts = len(self.aa_urls) if is_aa_auto_mode() else 1
|
||||
if self.attempts_this_dns >= max_attempts:
|
||||
@@ -1391,8 +1561,11 @@ class AAMirrorSelector:
|
||||
# Mirror is explicitly configured; do not fail over to other mirrors.
|
||||
return None, "exhausted"
|
||||
|
||||
if not self.aa_urls:
|
||||
return None, "exhausted"
|
||||
|
||||
next_index = (self._index + 1) % len(self.aa_urls)
|
||||
set_aa_url_index(next_index)
|
||||
set_aa_url(self.aa_urls[next_index])
|
||||
self._ensure_fresh_state(reset_attempts=False)
|
||||
return self.current_base, "mirror"
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Boot-time warm-up of the direct-download source.
|
||||
|
||||
The first AA search after a cold start pays for the whole cold path at once: DNS
|
||||
resolution, electing a live mirror, spinning up headless Chrome and solving the
|
||||
DDoS-Guard challenge. That is tens of seconds with the user sat at the search box.
|
||||
|
||||
Running one throwaway search shortly after boot moves that cost off the user's first
|
||||
search. It primes the DNS cache, elects (and quarantines) mirrors, and leaves the
|
||||
clearance cookie in the bypasser's per-domain cache, so the first real search reuses
|
||||
it instead of solving from scratch.
|
||||
|
||||
Runs on a daemon thread and swallows every failure: this is an optimisation, and a
|
||||
source that is down at boot must not affect startup or health.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Delay before the warm-up fires. Long enough that it does not compete with the rest
|
||||
# of startup (and with a container's own health probe) for the first request.
|
||||
_DEFAULT_DELAY_SECONDS = 15.0
|
||||
|
||||
_DEFAULT_QUERY = "The Great Gatsby"
|
||||
|
||||
_warmup_thread: threading.Thread | None = None
|
||||
_warmup_lock = threading.Lock()
|
||||
|
||||
|
||||
def _as_bool(value: object, *, default: bool) -> bool:
|
||||
"""Coerce a config value that may arrive as a string, bool or None."""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
from shelfmark.config.env import string_to_bool
|
||||
|
||||
return string_to_bool(value)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""Whether the boot-time warm-up search should run."""
|
||||
if not _as_bool(config.get("SEARCH_WARMUP_ENABLED", True), default=True):
|
||||
return False
|
||||
# Nothing to warm if the source is off, and no challenge to pre-solve without
|
||||
# the bypasser - a plain search is fast enough not to need this.
|
||||
if not _as_bool(config.get("DIRECT_DOWNLOAD_ENABLED", True), default=True):
|
||||
logger.debug("Search warm-up skipped: direct download disabled")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def warmup_query() -> str:
|
||||
"""The query used to warm the source."""
|
||||
raw = config.get("SEARCH_WARMUP_QUERY", _DEFAULT_QUERY)
|
||||
query = str(raw).strip() if raw else ""
|
||||
return query or _DEFAULT_QUERY
|
||||
|
||||
|
||||
def run_warmup() -> bool:
|
||||
"""Run one warm-up search. Returns True if it produced results.
|
||||
|
||||
Never raises: every failure mode here is one the next real search would hit
|
||||
anyway, and reporting it is the search path's job, not the warm-up's.
|
||||
"""
|
||||
from shelfmark.core.mirrors import has_aa_mirror_configuration
|
||||
|
||||
if not has_aa_mirror_configuration():
|
||||
logger.debug("Search warm-up skipped: no Anna's Archive mirrors configured")
|
||||
return False
|
||||
|
||||
query = warmup_query()
|
||||
logger.info("Warming up direct download search (%r)", query)
|
||||
try:
|
||||
from shelfmark.core.models import SearchFilters
|
||||
from shelfmark.release_sources.direct_download import search_books
|
||||
|
||||
results = search_books(query, SearchFilters())
|
||||
except Exception:
|
||||
# Broad by design: a warm-up must never take the app down, and the source
|
||||
# raises everything from network errors to parse failures.
|
||||
logger.warning("Search warm-up did not complete; first user search may be slow")
|
||||
logger.debug("Search warm-up failure detail", exc_info=True)
|
||||
return False
|
||||
|
||||
if results:
|
||||
logger.info("Search warm-up complete: %s results, source is ready", len(results))
|
||||
return True
|
||||
logger.info("Search warm-up returned no results; source reachable but empty")
|
||||
return False
|
||||
|
||||
|
||||
def start(delay_seconds: float = _DEFAULT_DELAY_SECONDS) -> bool:
|
||||
"""Schedule the warm-up on a daemon thread. Safe to call multiple times."""
|
||||
global _warmup_thread
|
||||
|
||||
if not is_enabled():
|
||||
return False
|
||||
|
||||
with _warmup_lock:
|
||||
if _warmup_thread is not None and _warmup_thread.is_alive():
|
||||
logger.debug("Search warm-up already scheduled")
|
||||
return False
|
||||
|
||||
def _run() -> None:
|
||||
run_warmup()
|
||||
|
||||
_warmup_thread = threading.Timer(delay_seconds, _run)
|
||||
_warmup_thread.daemon = True
|
||||
_warmup_thread.name = "SearchWarmup"
|
||||
_warmup_thread.start()
|
||||
|
||||
logger.debug("Search warm-up scheduled in %ss", delay_seconds)
|
||||
return True
|
||||
@@ -85,6 +85,7 @@ from shelfmark.core.requests_service import (
|
||||
from shelfmark.core.user_db import UserDB
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS, normalize_base_path
|
||||
from shelfmark.download import orchestrator as backend
|
||||
from shelfmark.download import warmup
|
||||
from shelfmark.release_sources import (
|
||||
BrowseRecord,
|
||||
Release,
|
||||
@@ -206,6 +207,10 @@ except (sqlite3.OperationalError, OSError) as e:
|
||||
# Start download coordinator
|
||||
backend.start()
|
||||
|
||||
# Pre-solve the direct-download source's protection challenge in the background so the
|
||||
# first user search does not pay for a cold Chrome bypass. Never blocks startup.
|
||||
warmup.start()
|
||||
|
||||
# Rate limiting for login attempts
|
||||
# Map usernames to their failed-attempt counters and lockout timestamps.
|
||||
failed_login_attempts: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@@ -539,6 +539,79 @@ class SearchUnavailableError(SourceUnavailableError):
|
||||
"""Raised when Anna's Archive cannot be reached via any mirror/DNS."""
|
||||
|
||||
|
||||
# Markers that prove a 200 really came from Anna's Archive, and markers that mean we
|
||||
# are looking at a protection interstitial rather than the site. A page with neither
|
||||
# is a domain that answers but is not AA - seized, parked or for sale.
|
||||
#
|
||||
# Deliberately structural rather than the domain name: a parking page's whole job is
|
||||
# to display the domain it is squatting on, so "annas-archive" matches the very pages
|
||||
# this is meant to catch. These paths only exist on the real site.
|
||||
_AA_PAGE_MARKERS = (
|
||||
"/md5/",
|
||||
"aarecord",
|
||||
"anna's archive",
|
||||
"/dyn/",
|
||||
"/datasets",
|
||||
"/fast_download",
|
||||
"/slow_download",
|
||||
)
|
||||
_CHALLENGE_MARKERS = (
|
||||
"ddos-guard",
|
||||
"just a moment",
|
||||
"cloudflare",
|
||||
"checking your browser",
|
||||
"cf-browser-verification",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_aa_page(html: str) -> bool:
|
||||
"""Whether ``html`` is recognisably Anna's Archive, or a challenge in front of it."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in (*_AA_PAGE_MARKERS, *_CHALLENGE_MARKERS))
|
||||
|
||||
|
||||
def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[str, Tag | None]:
|
||||
"""Fetch the AA search page, retrying past mirrors that are not actually AA.
|
||||
|
||||
A parked or seized domain answers 200 with a page that has no results table and no
|
||||
"No files found." - indistinguishable from a broken search unless we check whether
|
||||
the response looks like AA at all. Those mirrors are quarantined for the session so
|
||||
later searches skip them instead of paying the timeout again.
|
||||
"""
|
||||
attempt_url = url
|
||||
for _ in range(len(network.get_available_aa_urls()) or 1):
|
||||
response = downloader.html_get_page(
|
||||
attempt_url, selector=selector, allow_bypasser_fallback=True
|
||||
)
|
||||
if not response:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
|
||||
html = _html_response_text(response)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if isinstance(table, Tag):
|
||||
return html, table
|
||||
if table is not None:
|
||||
msg = f"Expected results table tag, got {type(table).__name__}"
|
||||
raise TypeError(msg)
|
||||
if "No files found." in html or _looks_like_aa_page(html):
|
||||
# A real AA response - either genuinely empty, or a shape the caller
|
||||
# should report as drift. Not the mirror's fault.
|
||||
return html, None
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=True, reason="responded without an Anna's Archive page"
|
||||
)
|
||||
if action not in ("mirror", "dns") or not new_base:
|
||||
return html, None
|
||||
attempt_url = selector.rewrite(url)
|
||||
logger.info("Retrying search on %s", new_base)
|
||||
|
||||
return "", None
|
||||
|
||||
|
||||
def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
"""Search for books matching the query.
|
||||
|
||||
@@ -603,15 +676,7 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
|
||||
# AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating
|
||||
# to another mirror only collects another 403, so let the bypasser solve it.
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True)
|
||||
if not html:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
|
||||
soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
tbody = soup.find("table")
|
||||
|
||||
html, tbody = _fetch_search_table(url, selector)
|
||||
if tbody is None:
|
||||
if "No files found." in html:
|
||||
logger.info("No books found for query: %s", query)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""A mirror that answers 200 with a non-AA page is quarantined, not reported as empty.
|
||||
|
||||
Seized and for-sale domains keep serving 200. Without a look at *what* came back, a
|
||||
parking page is indistinguishable from a broken search, so the mirror stays in
|
||||
rotation and every later search pays for it again.
|
||||
"""
|
||||
|
||||
from bs4 import Tag
|
||||
|
||||
PARKED_PAGE = """<!doctype html><html><head><title>annas-archive.li</title></head>
|
||||
<body><h1>This domain is for sale</h1><p>Inquire now. Buy this domain.</p></body></html>"""
|
||||
|
||||
AA_RESULTS_PAGE = """<!doctype html><html><body><main><table><tbody>
|
||||
<tr><td><a href="/md5/abc123"><img src="/c.jpg"></a></td><td><span>Dune</span></td></tr>
|
||||
</tbody></table></main></body></html>"""
|
||||
|
||||
AA_EMPTY_PAGE = """<!doctype html><html><body><main>
|
||||
<div>No files found.</div><a href="https://annas-archive.gl/about">about</a>
|
||||
</main></body></html>"""
|
||||
|
||||
DDOS_GUARD_PAGE = """<!doctype html><html><head><title>DDoS-Guard</title></head>
|
||||
<body><div id="ddos-guard">Checking your browser</div></body></html>"""
|
||||
|
||||
|
||||
class _Selector:
|
||||
def __init__(self, bases: list[str]) -> None:
|
||||
self._bases = bases
|
||||
self._index = 0
|
||||
self.current_base = bases[0]
|
||||
self.quarantined: list[str] = []
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
for base in self._bases:
|
||||
if url.startswith(base):
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, *, fatal: bool = False, reason: str = ""):
|
||||
if fatal:
|
||||
self.quarantined.append(self.current_base)
|
||||
self._index += 1
|
||||
if self._index >= len(self._bases):
|
||||
return None, "exhausted"
|
||||
self.current_base = self._bases[self._index]
|
||||
return self.current_base, "mirror"
|
||||
|
||||
|
||||
def _patch_pages(monkeypatch, pages: list[str]):
|
||||
"""Serve `pages` in order, recording the URL each call was made against."""
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_get(url, **_kwargs):
|
||||
calls.append(url)
|
||||
return pages[len(calls) - 1] if len(calls) <= len(pages) else ""
|
||||
|
||||
monkeypatch.setattr(dd.downloader, "html_get_page", fake_get)
|
||||
monkeypatch.setattr(dd.network, "get_available_aa_urls", lambda: ["a", "b", "c"])
|
||||
return dd, calls
|
||||
|
||||
|
||||
def test_parked_mirror_is_quarantined_and_search_retries_next_mirror(monkeypatch):
|
||||
dd, calls = _patch_pages(monkeypatch, [PARKED_PAGE, AA_RESULTS_PAGE])
|
||||
selector = _Selector(["https://parked.test", "https://real.test"])
|
||||
|
||||
html, table = dd._fetch_search_table("https://parked.test/search?q=dune", selector)
|
||||
|
||||
assert selector.quarantined == ["https://parked.test"]
|
||||
assert isinstance(table, Tag)
|
||||
assert "Dune" in html
|
||||
# The retry went to the live mirror, not back to the parked one.
|
||||
assert calls[1].startswith("https://real.test")
|
||||
|
||||
|
||||
def test_genuinely_empty_aa_result_does_not_quarantine(monkeypatch):
|
||||
"""'No files found.' is a real answer from a healthy mirror."""
|
||||
dd, _calls = _patch_pages(monkeypatch, [AA_EMPTY_PAGE])
|
||||
selector = _Selector(["https://real.test", "https://other.test"])
|
||||
|
||||
html, table = dd._fetch_search_table("https://real.test/search?q=zzz", selector)
|
||||
|
||||
assert selector.quarantined == []
|
||||
assert table is None
|
||||
assert "No files found." in html
|
||||
|
||||
|
||||
def test_challenge_page_does_not_quarantine(monkeypatch):
|
||||
"""A DDoS-Guard interstitial means the mirror is alive and holds our clearance."""
|
||||
dd, _calls = _patch_pages(monkeypatch, [DDOS_GUARD_PAGE])
|
||||
selector = _Selector(["https://real.test", "https://other.test"])
|
||||
|
||||
_html, table = dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
|
||||
assert selector.quarantined == []
|
||||
assert table is None
|
||||
|
||||
|
||||
def test_unreachable_mirror_raises_search_unavailable(monkeypatch):
|
||||
dd, _calls = _patch_pages(monkeypatch, [""])
|
||||
selector = _Selector(["https://real.test"])
|
||||
|
||||
try:
|
||||
dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
except dd.SearchUnavailableError:
|
||||
return
|
||||
raise AssertionError("expected SearchUnavailableError")
|
||||
@@ -0,0 +1,143 @@
|
||||
"""RFC 8484 wireformat codec.
|
||||
|
||||
Quad9 and OpenDNS reject the JSON API that Cloudflare and Google popularised, so
|
||||
these providers only work through wireformat. Quad9 additionally requires HTTP/2
|
||||
(section 5.2) and answers HTTP/1.1 with 505.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import struct
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.download import doh_wireformat as wf
|
||||
|
||||
|
||||
def _decode_param(param: str) -> bytes:
|
||||
padding = "=" * (-len(param) % 4)
|
||||
return base64.urlsafe_b64decode(param + padding)
|
||||
|
||||
|
||||
def _build_response(
|
||||
*, qname: str = "example.com", answers: list[tuple[int, bytes]], qtype: int = wf.TYPE_A
|
||||
) -> bytes:
|
||||
"""Assemble a response whose answer names are compression pointers to the question."""
|
||||
question = b""
|
||||
for label in qname.split("."):
|
||||
question += bytes([len(label)]) + label.encode()
|
||||
question += b"\x00" + struct.pack(">HH", qtype, 1)
|
||||
|
||||
body = b""
|
||||
for rtype, rdata in answers:
|
||||
body += b"\xc0\x0c" # pointer to offset 12 (the question name)
|
||||
body += struct.pack(">HHIH", rtype, 1, 300, len(rdata)) + rdata
|
||||
|
||||
header = struct.pack(">HHHHHH", 0, 0x8180, 1, len(answers), 0, 0)
|
||||
return header + question + body
|
||||
|
||||
|
||||
def test_encode_query_is_a_well_formed_dns_message():
|
||||
raw = _decode_param(wf.encode_query_param("example.com", wf.TYPE_A))
|
||||
|
||||
msg_id, flags, qdcount, ancount, _ns, _ar = struct.unpack_from(">HHHHHH", raw, 0)
|
||||
assert msg_id == 0 # RFC 8484 section 4.1: zero for cacheability
|
||||
assert flags == 0x0100 # recursion desired
|
||||
assert (qdcount, ancount) == (1, 0)
|
||||
assert raw[12:] == b"\x07example\x03com\x00" + struct.pack(">HH", wf.TYPE_A, 1)
|
||||
|
||||
|
||||
def test_encode_query_param_is_unpadded_base64url():
|
||||
param = wf.encode_query_param("example.com", wf.TYPE_A)
|
||||
assert "=" not in param
|
||||
assert "+" not in param and "/" not in param
|
||||
|
||||
|
||||
def test_encode_query_strips_trailing_dot():
|
||||
assert _decode_param(wf.encode_query_param("example.com.", wf.TYPE_A)) == _decode_param(
|
||||
wf.encode_query_param("example.com", wf.TYPE_A)
|
||||
)
|
||||
|
||||
|
||||
def test_encode_query_rejects_empty_hostname():
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.encode_query("", wf.TYPE_A)
|
||||
|
||||
|
||||
def test_encode_query_rejects_oversized_label():
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.encode_query("a" * 64 + ".com", wf.TYPE_A)
|
||||
|
||||
|
||||
def test_decode_a_records():
|
||||
response = _build_response(answers=[(wf.TYPE_A, bytes([93, 184, 216, 34]))])
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == ["93.184.216.34"]
|
||||
|
||||
|
||||
def test_decode_multiple_a_records_preserves_order():
|
||||
response = _build_response(
|
||||
answers=[(wf.TYPE_A, bytes([1, 1, 1, 1])), (wf.TYPE_A, bytes([8, 8, 8, 8]))]
|
||||
)
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == ["1.1.1.1", "8.8.8.8"]
|
||||
|
||||
|
||||
def test_decode_skips_cname_records_in_the_chain():
|
||||
"""Answers routinely lead with a CNAME; only the requested type is an address."""
|
||||
cname = b"\x03www\x07example\x03com\x00"
|
||||
response = _build_response(answers=[(5, cname), (wf.TYPE_A, bytes([93, 184, 216, 34]))])
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == ["93.184.216.34"]
|
||||
|
||||
|
||||
def test_decode_aaaa_compresses_zero_run():
|
||||
# 2606:4700:0:0:0:0:6810:84e5 -> the middle zero run collapses to "::"
|
||||
rdata = struct.pack(">8H", 0x2606, 0x4700, 0, 0, 0, 0, 0x6810, 0x84E5)
|
||||
response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA)
|
||||
assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2606:4700::6810:84e5"]
|
||||
|
||||
|
||||
def test_decode_aaaa_collapses_only_the_longest_zero_run():
|
||||
# 2001:0:0:1:0:0:0:1 - the second, longer run is the one that collapses.
|
||||
rdata = struct.pack(">8H", 0x2001, 0, 0, 1, 0, 0, 0, 1)
|
||||
response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA)
|
||||
assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2001:0:0:1::1"]
|
||||
|
||||
|
||||
def test_decode_aaaa_without_zero_run():
|
||||
rdata = struct.pack(">8H", 0x2001, 0x0DB8, 1, 2, 3, 4, 5, 6)
|
||||
response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA)
|
||||
assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2001:db8:1:2:3:4:5:6"]
|
||||
|
||||
|
||||
def test_decode_nxdomain_returns_empty_not_an_error():
|
||||
"""An empty answer is a valid response; the caller falls back rather than retrying."""
|
||||
response = _build_response(answers=[])
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == []
|
||||
|
||||
|
||||
def test_decode_rejects_truncated_header():
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.decode_answer(b"\x00\x01", wf.TYPE_A)
|
||||
|
||||
|
||||
def test_decode_rejects_truncated_record():
|
||||
response = _build_response(answers=[(wf.TYPE_A, bytes([1, 2, 3, 4]))])
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.decode_answer(response[:-2], wf.TYPE_A)
|
||||
|
||||
|
||||
def test_decode_does_not_hang_on_a_malicious_name():
|
||||
"""A self-referential name must not loop forever."""
|
||||
header = struct.pack(">HHHHHH", 0, 0x8180, 1, 0, 0, 0)
|
||||
# A run of maximum-length labels that never terminates.
|
||||
body = (b"\x3f" + b"a" * 63) * 8
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.decode_answer(header + body, wf.TYPE_A)
|
||||
|
||||
|
||||
def test_wireformat_providers_are_flagged(monkeypatch):
|
||||
"""The provider table and the resolver must agree on who needs wireformat."""
|
||||
import shelfmark.download.network as network
|
||||
|
||||
for name, servers, url in network.DNS_PROVIDERS:
|
||||
resolver = network.DoHResolver(url, "x.invalid", servers[0])
|
||||
expected = name in ("quad9", "opendns")
|
||||
assert resolver.use_wireformat is expected, f"{name} wireformat flag wrong"
|
||||
@@ -27,6 +27,7 @@ class _DummySelector:
|
||||
self._index = 0
|
||||
self.current_base = bases[0]
|
||||
self.attempts_this_dns = 0
|
||||
self.quarantined: list[tuple[str, str]] = []
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
for base in self._bases:
|
||||
@@ -34,7 +35,11 @@ class _DummySelector:
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, allow_dns: bool = True) -> tuple[str | None, str]:
|
||||
def next_mirror_or_rotate_dns(
|
||||
self, allow_dns: bool = True, *, fatal: bool = False, reason: str = ""
|
||||
) -> tuple[str | None, str]:
|
||||
if fatal:
|
||||
self.quarantined.append((self.current_base, reason))
|
||||
self.attempts_this_dns += 1
|
||||
self._index = (self._index + 1) % len(self._bases)
|
||||
self.current_base = self._bases[self._index]
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Tests for AA mirror quarantine: dead mirrors leave the rotation, live ones stay.
|
||||
|
||||
The distinction these guard is the whole point of the feature. A mirror that answers
|
||||
403 (DDoS-Guard) is alive and holds our bypass clearance, so rotating off it makes the
|
||||
next search solve a fresh challenge on a domain we have no cookie for. A mirror that
|
||||
NXDOMAINs, refuses the connection, or answers 200 with a parking page is not a mirror
|
||||
at all and must never be tried again this session.
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def _fresh_network(monkeypatch, urls: list[str], *, auto: bool = True):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "_initialized", True)
|
||||
monkeypatch.setattr(network, "_aa_urls", list(urls))
|
||||
monkeypatch.setattr(network, "_aa_base_url", urls[0])
|
||||
monkeypatch.setattr(network, "_current_aa_url_index", 0)
|
||||
monkeypatch.setattr(network, "_dead_aa_urls", set())
|
||||
monkeypatch.setattr(network, "_save_state", lambda **kwargs: None)
|
||||
monkeypatch.setattr(network, "is_aa_auto_mode", lambda: auto)
|
||||
return network
|
||||
|
||||
|
||||
MIRRORS = ["https://aa-one.test", "https://aa-two.test", "https://aa-three.test"]
|
||||
|
||||
|
||||
def test_quarantined_mirror_leaves_the_available_list(monkeypatch):
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True
|
||||
assert network.get_available_aa_urls() == ["https://aa-one.test", "https://aa-three.test"]
|
||||
assert network.get_dead_aa_urls() == {"https://aa-two.test"}
|
||||
|
||||
|
||||
def test_quarantine_accepts_a_full_request_url(monkeypatch):
|
||||
"""Callers hold the failing request URL, not the bare mirror base."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-two.test/search?q=dune", "parked") is True
|
||||
assert "https://aa-two.test" in network.get_dead_aa_urls()
|
||||
|
||||
|
||||
def test_quarantine_is_idempotent(monkeypatch):
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is False
|
||||
assert network.get_available_aa_urls() == ["https://aa-one.test", "https://aa-three.test"]
|
||||
|
||||
|
||||
def test_last_surviving_mirror_is_never_quarantined(monkeypatch):
|
||||
"""Misclassification must not leave the app with nowhere to search."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-one.test", "NXDOMAIN") is True
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True
|
||||
assert network.mark_aa_url_dead("https://aa-three.test", "NXDOMAIN") is False
|
||||
assert network.get_available_aa_urls() == ["https://aa-three.test"]
|
||||
|
||||
|
||||
def test_selector_skips_quarantined_mirror_when_rotating(monkeypatch):
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
selector = network.AAMirrorSelector()
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(fatal=True, reason="NXDOMAIN")
|
||||
|
||||
assert action == "mirror"
|
||||
# Landed on the next live mirror, not skipped past it onto the third.
|
||||
assert new_base == "https://aa-two.test"
|
||||
assert "https://aa-one.test" in network.get_dead_aa_urls()
|
||||
assert selector.rewrite("https://aa-one.test/search") == "https://aa-two.test/search"
|
||||
|
||||
|
||||
def test_non_fatal_rotation_keeps_the_mirror(monkeypatch):
|
||||
"""A 5xx or a challenge rotates but must not burn the mirror."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
selector = network.AAMirrorSelector()
|
||||
|
||||
selector.next_mirror_or_rotate_dns()
|
||||
|
||||
assert network.get_dead_aa_urls() == set()
|
||||
assert network.get_available_aa_urls() == MIRRORS
|
||||
|
||||
|
||||
def test_dns_reset_does_not_resurrect_quarantined_mirrors(monkeypatch):
|
||||
"""A new DNS provider cannot revive a parked domain, so it stays skipped."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
monkeypatch.setattr(network, "rotate_dns_provider", lambda: True)
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
network.mark_aa_url_dead("https://aa-one.test", "parked")
|
||||
|
||||
assert network.rotate_dns_and_reset_aa() is True
|
||||
assert network.get_aa_base_url() == "https://aa-two.test"
|
||||
|
||||
|
||||
def test_editing_the_mirror_list_clears_quarantine(monkeypatch):
|
||||
"""Quarantine decisions were made about a list the user has now changed."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-two.test", "parked")
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: [*MIRRORS, "https://aa-four.test"])
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"})
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert network.get_dead_aa_urls() == set()
|
||||
|
||||
|
||||
def test_reinit_with_an_unchanged_list_keeps_quarantine(monkeypatch):
|
||||
"""Re-init happens constantly (settings sync, DNS rotation, helper startup).
|
||||
|
||||
Clearing quarantine on every one of those resurrects a parked mirror mid-session,
|
||||
which is exactly the bug this guards: the mirror gets re-elected and the next
|
||||
search pays for it again.
|
||||
"""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-two.test", "parked")
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS))
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"})
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert network.get_dead_aa_urls() == {"https://aa-two.test"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Failure classification
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _http_error(status: int) -> requests.exceptions.HTTPError:
|
||||
response = requests.Response()
|
||||
response.status_code = status
|
||||
return requests.exceptions.HTTPError(response=response)
|
||||
|
||||
|
||||
def test_dns_failure_is_fatal_for_the_mirror():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
exc = requests.exceptions.ConnectionError(
|
||||
"HTTPSConnectionPool(host='aa.test', port=443): Max retries exceeded "
|
||||
"(Caused by NameResolutionError(\"Failed to resolve 'aa.test'\"))"
|
||||
)
|
||||
assert http._fatal_mirror_reason(exc) == "DNS does not resolve"
|
||||
|
||||
|
||||
def test_connection_refused_is_fatal_for_the_mirror():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
exc = requests.exceptions.ConnectionError("Connection refused")
|
||||
assert http._fatal_mirror_reason(exc) == "connection refused"
|
||||
|
||||
|
||||
def test_gone_and_legal_block_are_fatal():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
assert http._fatal_mirror_reason(_http_error(410)) == "HTTP 410"
|
||||
assert http._fatal_mirror_reason(_http_error(451)) == "HTTP 451"
|
||||
|
||||
|
||||
def test_timeout_is_not_fatal():
|
||||
"""A slow mirror is still a mirror - and may hold our bypass clearance."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
assert http._fatal_mirror_reason(requests.exceptions.ConnectTimeout("timed out")) is None
|
||||
assert http._fatal_mirror_reason(requests.exceptions.ReadTimeout("timed out")) is None
|
||||
|
||||
|
||||
def test_challenge_and_server_errors_are_not_fatal():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
for status in (403, 429, 500, 502, 503):
|
||||
assert http._fatal_mirror_reason(_http_error(status)) is None
|
||||
|
||||
|
||||
def test_startup_probe_skips_quarantined_mirrors(monkeypatch):
|
||||
"""Re-init must not re-probe (or re-elect) a mirror already known to be dead.
|
||||
|
||||
A parking page answers 200, so an unfiltered probe elects it every single time
|
||||
the app re-initialises - one wasted request per re-init, forever.
|
||||
"""
|
||||
import requests
|
||||
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-one.test", "parked")
|
||||
probed: list[str] = []
|
||||
|
||||
def fake_get(url, **_kwargs):
|
||||
probed.append(url)
|
||||
response = requests.Response()
|
||||
response.status_code = 200
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(network.requests, "get", fake_get)
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS))
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {})
|
||||
monkeypatch.setattr(network, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(network, "get_ssl_verify", lambda _url: True)
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert "https://aa-one.test" not in probed
|
||||
assert network.get_aa_base_url() == "https://aa-two.test"
|
||||
|
||||
|
||||
def test_startup_probe_does_not_restore_a_quarantined_mirror(monkeypatch):
|
||||
"""Saved state can name a mirror that has since been quarantined."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-one.test", "parked")
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS))
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"})
|
||||
monkeypatch.setattr(network, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(network, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(network.requests, "get", lambda *a, **k: (_ for _ in ()).throw(OSError()))
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert network.get_aa_base_url() != "https://aa-one.test"
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Boot-time search warm-up.
|
||||
|
||||
The warm-up exists to move the cold DDoS-Guard solve off the user's first search. It
|
||||
is an optimisation, so the load-bearing property is that it can never affect startup:
|
||||
a source that is down, misconfigured or raising must leave the app running.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def warmup(monkeypatch):
|
||||
import shelfmark.download.warmup as warmup_module
|
||||
|
||||
monkeypatch.setattr(warmup_module, "_warmup_thread", None)
|
||||
return warmup_module
|
||||
|
||||
|
||||
def _patch_config(monkeypatch, warmup, values: dict):
|
||||
def fake_get(key, default=None):
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(warmup.config, "get", fake_get)
|
||||
|
||||
|
||||
def test_disabled_by_setting(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": False})
|
||||
assert warmup.is_enabled() is False
|
||||
assert warmup.start() is False
|
||||
|
||||
|
||||
def test_disabled_by_string_false(monkeypatch, warmup):
|
||||
"""Deployment ENV arrives as a string, not a bool."""
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": "false"})
|
||||
assert warmup.is_enabled() is False
|
||||
|
||||
|
||||
def test_skipped_when_direct_download_is_off(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {"DIRECT_DOWNLOAD_ENABLED": False})
|
||||
assert warmup.is_enabled() is False
|
||||
|
||||
|
||||
def test_enabled_by_default(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
assert warmup.is_enabled() is True
|
||||
|
||||
|
||||
def test_query_defaults_and_is_configurable(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
assert warmup.warmup_query() == "The Great Gatsby"
|
||||
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "Dune"})
|
||||
assert warmup.warmup_query() == "Dune"
|
||||
|
||||
# A blank override must not send an empty query at the source.
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": " "})
|
||||
assert warmup.warmup_query() == "The Great Gatsby"
|
||||
|
||||
|
||||
def test_skipped_when_no_mirrors_configured(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: False)
|
||||
|
||||
called: list[str] = []
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(dd, "search_books", lambda q, f: called.append(q))
|
||||
|
||||
assert warmup.run_warmup() is False
|
||||
assert called == []
|
||||
|
||||
|
||||
def test_successful_warmup_reports_true(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
|
||||
seen: list[str] = []
|
||||
|
||||
def fake_search(query, _filters):
|
||||
seen.append(query)
|
||||
return ["a", "b"]
|
||||
|
||||
monkeypatch.setattr(dd, "search_books", fake_search)
|
||||
|
||||
assert warmup.run_warmup() is True
|
||||
assert seen == ["The Great Gatsby"]
|
||||
|
||||
|
||||
def test_empty_results_are_not_an_error(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
|
||||
monkeypatch.setattr(dd, "search_books", lambda q, f: [])
|
||||
|
||||
assert warmup.run_warmup() is False
|
||||
|
||||
|
||||
def test_search_failure_is_swallowed(monkeypatch, warmup):
|
||||
"""A source that is down at boot must not propagate out of the warm-up."""
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
|
||||
|
||||
def boom(_query, _filters):
|
||||
msg = "mirrors are blocked"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
monkeypatch.setattr(dd, "search_books", boom)
|
||||
|
||||
assert warmup.run_warmup() is False
|
||||
|
||||
|
||||
def test_start_schedules_a_daemon_thread_and_is_idempotent(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
|
||||
assert warmup.start(delay_seconds=30) is True
|
||||
thread = warmup._warmup_thread
|
||||
assert thread is not None
|
||||
assert thread.daemon is True
|
||||
|
||||
# A second call must not stack up another timer.
|
||||
assert warmup.start(delay_seconds=30) is False
|
||||
assert warmup._warmup_thread is thread
|
||||
|
||||
thread.cancel()
|
||||
|
||||
|
||||
def test_start_does_not_run_the_search_inline(monkeypatch, warmup):
|
||||
"""Startup must not block on a search that can take a minute."""
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
ran: list[bool] = []
|
||||
monkeypatch.setattr(warmup, "run_warmup", lambda: ran.append(True))
|
||||
|
||||
warmup.start(delay_seconds=30)
|
||||
assert ran == []
|
||||
|
||||
if warmup._warmup_thread:
|
||||
warmup._warmup_thread.cancel()
|
||||
@@ -1,7 +1,19 @@
|
||||
version = 1
|
||||
revision = 3
|
||||
revision = 2
|
||||
requires-python = ">=3.14"
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "apprise"
|
||||
version = "1.12.0"
|
||||
@@ -546,6 +558,70 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "4.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "hpack" },
|
||||
{ name = "hyperframe" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hpack"
|
||||
version = "4.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "1.0.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "h11" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpx"
|
||||
version = "0.28.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "certifi" },
|
||||
{ name = "httpcore" },
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
http2 = [
|
||||
{ name = "h2" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyperframe"
|
||||
version = "6.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.18"
|
||||
@@ -1391,6 +1467,7 @@ dependencies = [
|
||||
{ name = "gevent" },
|
||||
{ name = "gevent-websocket" },
|
||||
{ name = "gunicorn" },
|
||||
{ name = "httpx", extra = ["http2"] },
|
||||
{ name = "psutil" },
|
||||
{ name = "python-socketio" },
|
||||
{ name = "qbittorrent-api" },
|
||||
@@ -1433,6 +1510,7 @@ requires-dist = [
|
||||
{ name = "gevent" },
|
||||
{ name = "gevent-websocket" },
|
||||
{ name = "gunicorn" },
|
||||
{ name = "httpx", extras = ["http2"], specifier = ">=0.27" },
|
||||
{ name = "psutil" },
|
||||
{ name = "pyautogui", marker = "extra == 'browser'" },
|
||||
{ name = "python-socketio" },
|
||||
|
||||
Reference in New Issue
Block a user