fix(download): hand a 503 carrying a challenge to the bypasser (#1238)

503 is in RETRYABLE_CODES, and the bypasser is only ever reached from
the 403
branch and the AA redirect-loop rescues. Once Z-Library re-serves its
DDoS-Guard
interstitial with the same cookie the #1188 handshake already echoed
back, the
request has nothing left to try and spends every attempt on the same
wall.

Gate the handoff on the response body rather than the status, so a
genuine
overloaded-origin 503 keeps its retry path, and on
allow_bypasser_fallback, so
best-effort fetches still fail fast. The challenge indicators move out
of
internal_bypasser into shelfmark/bypass/challenge.py so http.py can use
them
without importing SeleniumBase, which is lazily imported precisely
because it
is optional.

Refs #1233

shelfmark/bypass/challenge.py

@@ -0,0 +1,52 @@
+"""Challenge-page detection shared by the bypassers and the HTTP retry path.
+
+Kept out of `internal_bypasser` so the HTTP layer can recognise an interstitial
+without importing SeleniumBase: that module is imported lazily precisely because its
+browser dependencies are optional, and external-bypasser setups run without them.
+"""
+
+# Matched against lowercased text, so every entry must be lowercase.
+CLOUDFLARE_INDICATORS = [
+    "just a moment",
+    "verify you are human",
+    "verifying you are human",
+    "cloudflare.com/products/turnstile",
+]
+
+DDOS_GUARD_INDICATORS = [
+    "ddos-guard",
+    "ddos guard",
+    "checking your browser before accessing",
+    "complete the manual check to continue",
+    "could not verify your browser automatically",
+]
+
+# Markers that exist only in raw markup: the bypassers scan rendered innerText, where
+# a script src or a <title> never appears. The title match is scoped to the tag on
+# purpose - hosts word the rest of that sentence differently, and matching "checking
+# your browser" as free text would trip on any page that merely discusses a challenge.
+_RAW_HTML_MARKERS = (
+    "<title>checking your browser",
+    "/cdn-cgi/challenge-platform",
+    "/.well-known/ddos-guard/",
+)
+
+# An interstitial is a few KB of markup. Past that it is a real page that happens to
+# mention a marker - a protected site links its own DDoS-Guard endpoints on every page.
+MAX_CHALLENGE_HTML_CHARS = 64 * 1024
+
+
+def challenge_marker(html: str) -> str | None:
+    """Return the marker proving `html` is an unsolved challenge page, or None.
+
+    Only meaningful for a response that already carries a challenge status: the
+    markers appear on protected sites' real pages too, so the status is what
+    separates "blocked" from "served".
+    """
+    if not html or len(html) > MAX_CHALLENGE_HTML_CHARS:
+        return None
+    lowered = html.lower()
+    for marker in (*_RAW_HTML_MARKERS, *DDOS_GUARD_INDICATORS, *CLOUDFLARE_INDICATORS):
+        if marker in lowered:
+            return marker
+    return None

shelfmark/bypass/internal_bypasser.py

@@ -27,6 +27,7 @@ from seleniumbase import cdp_driver
 from seleniumbase.undetected.cdp_driver.connection import ProtocolException
 
 from shelfmark.bypass import BypassCancelledError
+from shelfmark.bypass.challenge import CLOUDFLARE_INDICATORS, DDOS_GUARD_INDICATORS
 from shelfmark.bypass.cookie_store import (
     clear_cf_cookies,
     export_store,
@@ -63,22 +64,6 @@ _IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS
 _BYPASS_CHILD_ENV = "SHELFMARK_INTERNAL_BYPASSER_CHILD"
 _PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0
 
-# Challenge detection indicators
-CLOUDFLARE_INDICATORS = [
-    "just a moment",
-    "verify you are human",
-    "verifying you are human",
-    "cloudflare.com/products/turnstile",
-]
-
-DDOS_GUARD_INDICATORS = [
-    "ddos-guard",
-    "ddos guard",
-    "checking your browser before accessing",
-    "complete the manual check to continue",
-    "could not verify your browser automatically",
-]
-
 
 class _DisplayState(TypedDict):
     ffmpeg: subprocess.Popen[bytes] | None

shelfmark/download/http.py

@@ -11,6 +11,7 @@ import requests
 from tqdm import tqdm
 
 from shelfmark.bypass import BypassCancelledError, cookie_store
+from shelfmark.bypass.challenge import challenge_marker
 from shelfmark.core.config import config as app_config
 from shelfmark.core.logger import setup_logger
 from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int
@@ -233,6 +234,22 @@ def _is_retryable_error(e: Exception) -> bool:
 _DEAD_MIRROR_CODES = (410, 451)
 
 
+def _response_challenge_marker(response: requests.Response) -> str | None:
+    """The challenge marker in a response body, or None if it carries no challenge.
+
+    Content type is checked first so a JSON or octet-stream error body is never
+    decoded just to be scanned; a missing header is scanned anyway, since an
+    interstitial served without one is still an interstitial.
+    """
+    content_type = response.headers.get("Content-Type", "")
+    if content_type and "html" not in content_type.lower():
+        return None
+    try:
+        return challenge_marker(response.text)
+    except UnicodeDecodeError, ValueError:
+        return None
+
+
 def _fatal_mirror_reason(e: Exception) -> str | None:
     """Return why ``e`` proves the mirror is unusable, or None if it may recover.
 
@@ -455,6 +472,35 @@ def html_get_page(
                         )
                         continue
 
+                # A 503 still serving a challenge is protection, not a busy origin. The
+                # handshake above has nothing left to echo back, and 503 is in
+                # RETRYABLE_CODES, so without this the request spends every attempt on
+                # the same wall: the bypasser is only ever reached from the 403 branch
+                # and the AA redirect rescues. Gate on the body, not the status, so a
+                # genuine overloaded-origin 503 keeps its retry path.
+                if response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE:
+                    marker = _response_challenge_marker(response)
+                    if marker and _bypass_handoff_allowed():
+                        if cookies:
+                            # Challenged while presenting clearance means those cookies
+                            # are dead; same reasoning as the 403 branch below.
+                            logger.debug(
+                                "503 challenge with cookies presented; purging: %s", current_url
+                            )
+                            _purge_clearance(current_url)
+                        logger.info(
+                            "503 challenge detected (%s); switching to bypasser: %s",
+                            marker,
+                            current_url,
+                        )
+                        return _run_bypasser(current_url)
+                    if marker:
+                        logger.debug(
+                            "503 challenge (%s) but no bypasser handoff available: %s",
+                            marker,
+                            current_url,
+                        )
+
                 if is_aa_url and response.is_redirect:
                     location = response.headers.get("Location", "")
                     if not location:

tests/download/test_http_challenge_503.py

@@ -0,0 +1,127 @@
+"""Tests for handing a 503 that carries a browser challenge to the bypasser."""
+
+import requests
+
+_CHALLENGE_HTML = (
+    "<html><head><title>Checking your browser before accessing z-lib.gd</title>"
+    "<script src='/.well-known/ddos-guard/check.js'></script></head>"
+    "<body>Please wait...</body></html>"
+)
+
+
+class _FakeResponse:
+    """Minimal stand-in for requests.Response covering what html_get_page touches."""
+
+    def __init__(
+        self,
+        status_code: int,
+        *,
+        url: str = "https://z-lib.gd/md5/abc",
+        text: str = "",
+        cookies: dict[str, str] | None = None,
+    ) -> None:
+        self.status_code = status_code
+        self.url = url
+        self.text = text
+        self.cookies = cookies or {}
+        self.headers = {"Content-Type": "text/html;charset=utf-8"}
+        self.is_redirect = False
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            error = requests.exceptions.HTTPError(f"{self.status_code} Error")
+            error.response = self
+            raise error
+
+
+def _neutralize_network(monkeypatch, http):
+    monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
+    monkeypatch.setattr(http, "get_proxies", lambda _url: {})
+    monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
+    monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
+    monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
+
+
+def test_503_challenge_is_handed_to_the_bypasser(monkeypatch):
+    """The reissued-cookie 503 from #1233 reaches the bypasser instead of retrying."""
+    import shelfmark.download.http as http
+
+    _neutralize_network(monkeypatch, http)
+    monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
+
+    attempts: list[dict[str, str]] = []
+    bypassed: list[str] = []
+
+    def fake_get(_url: str, **kwargs):
+        attempts.append(dict(kwargs["cookies"]))
+        # Hit 1 issues the cookie; every later hit re-serves the challenge unchanged,
+        # which is what leaves the handshake with nothing to echo back.
+        if len(attempts) == 1:
+            return _FakeResponse(503, cookies={"bsrv": "1"})
+        return _FakeResponse(503, text=_CHALLENGE_HTML, cookies={"bsrv": "1"})
+
+    def fake_bypass(url: str, _selector=None, _cancel_flag=None):
+        bypassed.append(url)
+        return "<html>real page</html>"
+
+    monkeypatch.setattr(http.requests, "get", fake_get)
+    monkeypatch.setattr(http, "get_bypassed_page", fake_bypass)
+
+    html = http.html_get_page("https://z-lib.gd/md5/abc", retry=10, success_delay=0)
+
+    assert html == "<html>real page</html>"
+    assert bypassed == ["https://z-lib.gd/md5/abc"]
+    # The handshake still gets its echo; the challenge ends the loop on the second hit
+    # rather than burning all ten attempts.
+    assert attempts == [{}, {"bsrv": "1"}]
+
+
+def test_plain_503_still_retries_without_bypassing(monkeypatch):
+    """An overloaded origin has no challenge marker, so its retry path is untouched."""
+    import shelfmark.download.http as http
+
+    _neutralize_network(monkeypatch, http)
+    monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
+
+    attempts: list[dict[str, str]] = []
+    bypassed: list[str] = []
+
+    def fake_get(_url: str, **kwargs):
+        attempts.append(dict(kwargs["cookies"]))
+        return _FakeResponse(503, text="<html><body>Service Unavailable</body></html>")
+
+    monkeypatch.setattr(http.requests, "get", fake_get)
+    monkeypatch.setattr(
+        http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or ""
+    )
+
+    html = http.html_get_page("https://z-lib.gd/md5/abc", retry=3, success_delay=0)
+
+    assert html == ""
+    assert bypassed == []
+    assert attempts == [{}, {}, {}]
+
+
+def test_503_challenge_respects_disabled_bypasser_fallback(monkeypatch):
+    """Best-effort fetches must not stall on a minutes-long solve."""
+    import shelfmark.download.http as http
+
+    _neutralize_network(monkeypatch, http)
+    monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
+
+    bypassed: list[str] = []
+    monkeypatch.setattr(
+        http.requests,
+        "get",
+        lambda _url, **_kwargs: _FakeResponse(503, text=_CHALLENGE_HTML),
+    )
+    monkeypatch.setattr(
+        http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or ""
+    )
+
+    html = http.html_get_page(
+        "https://z-lib.gd/md5/abc", retry=1, success_delay=0, allow_bypasser_fallback=False
+    )
+
+    assert html == ""
+    assert bypassed == []