From 12d554a92f1829dedadfe9829b7d2f8f4b0e4e3a Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Thu, 20 Aug 2026 10:29:52 -0400 Subject: [PATCH] fix(download): hand a 503 carrying a challenge to the bypasser (#1238) 503 is in RETRYABLE_CODES, and the bypasser is only ever reached from the 403 branch and the AA redirect-loop rescues. Once Z-Library re-serves its DDoS-Guard interstitial with the same cookie the #1188 handshake already echoed back, the request has nothing left to try and spends every attempt on the same wall. Gate the handoff on the response body rather than the status, so a genuine overloaded-origin 503 keeps its retry path, and on allow_bypasser_fallback, so best-effort fetches still fail fast. The challenge indicators move out of internal_bypasser into shelfmark/bypass/challenge.py so http.py can use them without importing SeleniumBase, which is lazily imported precisely because it is optional. Refs #1233 --- shelfmark/bypass/challenge.py | 52 +++++++++ shelfmark/bypass/internal_bypasser.py | 17 +-- shelfmark/download/http.py | 46 ++++++++ tests/download/test_http_challenge_503.py | 127 ++++++++++++++++++++++ 4 files changed, 226 insertions(+), 16 deletions(-) create mode 100644 shelfmark/bypass/challenge.py create mode 100644 tests/download/test_http_challenge_503.py diff --git a/shelfmark/bypass/challenge.py b/shelfmark/bypass/challenge.py new file mode 100644 index 0000000..b24a022 --- /dev/null +++ b/shelfmark/bypass/challenge.py @@ -0,0 +1,52 @@ +"""Challenge-page detection shared by the bypassers and the HTTP retry path. + +Kept out of `internal_bypasser` so the HTTP layer can recognise an interstitial +without importing SeleniumBase: that module is imported lazily precisely because its +browser dependencies are optional, and external-bypasser setups run without them. +""" + +# Matched against lowercased text, so every entry must be lowercase. +CLOUDFLARE_INDICATORS = [ + "just a moment", + "verify you are human", + "verifying you are human", + "cloudflare.com/products/turnstile", +] + +DDOS_GUARD_INDICATORS = [ + "ddos-guard", + "ddos guard", + "checking your browser before accessing", + "complete the manual check to continue", + "could not verify your browser automatically", +] + +# Markers that exist only in raw markup: the bypassers scan rendered innerText, where +# a script src or a never appears. The title match is scoped to the tag on +# purpose - hosts word the rest of that sentence differently, and matching "checking +# your browser" as free text would trip on any page that merely discusses a challenge. +_RAW_HTML_MARKERS = ( + "<title>checking your browser", + "/cdn-cgi/challenge-platform", + "/.well-known/ddos-guard/", +) + +# An interstitial is a few KB of markup. Past that it is a real page that happens to +# mention a marker - a protected site links its own DDoS-Guard endpoints on every page. +MAX_CHALLENGE_HTML_CHARS = 64 * 1024 + + +def challenge_marker(html: str) -> str | None: + """Return the marker proving `html` is an unsolved challenge page, or None. + + Only meaningful for a response that already carries a challenge status: the + markers appear on protected sites' real pages too, so the status is what + separates "blocked" from "served". + """ + if not html or len(html) > MAX_CHALLENGE_HTML_CHARS: + return None + lowered = html.lower() + for marker in (*_RAW_HTML_MARKERS, *DDOS_GUARD_INDICATORS, *CLOUDFLARE_INDICATORS): + if marker in lowered: + return marker + return None diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index 429820e..647231f 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -27,6 +27,7 @@ from seleniumbase import cdp_driver from seleniumbase.undetected.cdp_driver.connection import ProtocolException from shelfmark.bypass import BypassCancelledError +from shelfmark.bypass.challenge import CLOUDFLARE_INDICATORS, DDOS_GUARD_INDICATORS from shelfmark.bypass.cookie_store import ( clear_cf_cookies, export_store, @@ -63,22 +64,6 @@ _IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS _BYPASS_CHILD_ENV = "SHELFMARK_INTERNAL_BYPASSER_CHILD" _PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0 -# Challenge detection indicators -CLOUDFLARE_INDICATORS = [ - "just a moment", - "verify you are human", - "verifying you are human", - "cloudflare.com/products/turnstile", -] - -DDOS_GUARD_INDICATORS = [ - "ddos-guard", - "ddos guard", - "checking your browser before accessing", - "complete the manual check to continue", - "could not verify your browser automatically", -] - class _DisplayState(TypedDict): ffmpeg: subprocess.Popen[bytes] | None diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index e010e35..2b5afce 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -11,6 +11,7 @@ import requests from tqdm import tqdm from shelfmark.bypass import BypassCancelledError, cookie_store +from shelfmark.bypass.challenge import challenge_marker from shelfmark.core.config import config as app_config from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int @@ -233,6 +234,22 @@ def _is_retryable_error(e: Exception) -> bool: _DEAD_MIRROR_CODES = (410, 451) +def _response_challenge_marker(response: requests.Response) -> str | None: + """The challenge marker in a response body, or None if it carries no challenge. + + Content type is checked first so a JSON or octet-stream error body is never + decoded just to be scanned; a missing header is scanned anyway, since an + interstitial served without one is still an interstitial. + """ + content_type = response.headers.get("Content-Type", "") + if content_type and "html" not in content_type.lower(): + return None + try: + return challenge_marker(response.text) + except UnicodeDecodeError, ValueError: + return None + + def _fatal_mirror_reason(e: Exception) -> str | None: """Return why ``e`` proves the mirror is unusable, or None if it may recover. @@ -455,6 +472,35 @@ def html_get_page( ) continue + # A 503 still serving a challenge is protection, not a busy origin. The + # handshake above has nothing left to echo back, and 503 is in + # RETRYABLE_CODES, so without this the request spends every attempt on + # the same wall: the bypasser is only ever reached from the 403 branch + # and the AA redirect rescues. Gate on the body, not the status, so a + # genuine overloaded-origin 503 keeps its retry path. + if response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE: + marker = _response_challenge_marker(response) + if marker and _bypass_handoff_allowed(): + if cookies: + # Challenged while presenting clearance means those cookies + # are dead; same reasoning as the 403 branch below. + logger.debug( + "503 challenge with cookies presented; purging: %s", current_url + ) + _purge_clearance(current_url) + logger.info( + "503 challenge detected (%s); switching to bypasser: %s", + marker, + current_url, + ) + return _run_bypasser(current_url) + if marker: + logger.debug( + "503 challenge (%s) but no bypasser handoff available: %s", + marker, + current_url, + ) + if is_aa_url and response.is_redirect: location = response.headers.get("Location", "") if not location: diff --git a/tests/download/test_http_challenge_503.py b/tests/download/test_http_challenge_503.py new file mode 100644 index 0000000..dbb544c --- /dev/null +++ b/tests/download/test_http_challenge_503.py @@ -0,0 +1,127 @@ +"""Tests for handing a 503 that carries a browser challenge to the bypasser.""" + +import requests + +_CHALLENGE_HTML = ( + "<html><head><title>Checking your browser before accessing z-lib.gd" + "" + "Please wait..." +) + + +class _FakeResponse: + """Minimal stand-in for requests.Response covering what html_get_page touches.""" + + def __init__( + self, + status_code: int, + *, + url: str = "https://z-lib.gd/md5/abc", + text: str = "", + cookies: dict[str, str] | None = None, + ) -> None: + self.status_code = status_code + self.url = url + self.text = text + self.cookies = cookies or {} + self.headers = {"Content-Type": "text/html;charset=utf-8"} + self.is_redirect = False + + def raise_for_status(self) -> None: + if self.status_code >= 400: + error = requests.exceptions.HTTPError(f"{self.status_code} Error") + error.response = self + raise error + + +def _neutralize_network(monkeypatch, http): + monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {}) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True) + monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False) + monkeypatch.setattr(http.time, "sleep", lambda _seconds: None) + + +def test_503_challenge_is_handed_to_the_bypasser(monkeypatch): + """The reissued-cookie 503 from #1233 reaches the bypasser instead of retrying.""" + import shelfmark.download.http as http + + _neutralize_network(monkeypatch, http) + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + attempts: list[dict[str, str]] = [] + bypassed: list[str] = [] + + def fake_get(_url: str, **kwargs): + attempts.append(dict(kwargs["cookies"])) + # Hit 1 issues the cookie; every later hit re-serves the challenge unchanged, + # which is what leaves the handshake with nothing to echo back. + if len(attempts) == 1: + return _FakeResponse(503, cookies={"bsrv": "1"}) + return _FakeResponse(503, text=_CHALLENGE_HTML, cookies={"bsrv": "1"}) + + def fake_bypass(url: str, _selector=None, _cancel_flag=None): + bypassed.append(url) + return "real page" + + monkeypatch.setattr(http.requests, "get", fake_get) + monkeypatch.setattr(http, "get_bypassed_page", fake_bypass) + + html = http.html_get_page("https://z-lib.gd/md5/abc", retry=10, success_delay=0) + + assert html == "real page" + assert bypassed == ["https://z-lib.gd/md5/abc"] + # The handshake still gets its echo; the challenge ends the loop on the second hit + # rather than burning all ten attempts. + assert attempts == [{}, {"bsrv": "1"}] + + +def test_plain_503_still_retries_without_bypassing(monkeypatch): + """An overloaded origin has no challenge marker, so its retry path is untouched.""" + import shelfmark.download.http as http + + _neutralize_network(monkeypatch, http) + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + attempts: list[dict[str, str]] = [] + bypassed: list[str] = [] + + def fake_get(_url: str, **kwargs): + attempts.append(dict(kwargs["cookies"])) + return _FakeResponse(503, text="Service Unavailable") + + monkeypatch.setattr(http.requests, "get", fake_get) + monkeypatch.setattr( + http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or "" + ) + + html = http.html_get_page("https://z-lib.gd/md5/abc", retry=3, success_delay=0) + + assert html == "" + assert bypassed == [] + assert attempts == [{}, {}, {}] + + +def test_503_challenge_respects_disabled_bypasser_fallback(monkeypatch): + """Best-effort fetches must not stall on a minutes-long solve.""" + import shelfmark.download.http as http + + _neutralize_network(monkeypatch, http) + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + bypassed: list[str] = [] + monkeypatch.setattr( + http.requests, + "get", + lambda _url, **_kwargs: _FakeResponse(503, text=_CHALLENGE_HTML), + ) + monkeypatch.setattr( + http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or "" + ) + + html = http.html_get_page( + "https://z-lib.gd/md5/abc", retry=1, success_delay=0, allow_bypasser_fallback=False + ) + + assert html == "" + assert bypassed == []