diff --git a/shelfmark/bypass/challenge.py b/shelfmark/bypass/challenge.py new file mode 100644 index 0000000..b24a022 --- /dev/null +++ b/shelfmark/bypass/challenge.py @@ -0,0 +1,52 @@ +"""Challenge-page detection shared by the bypassers and the HTTP retry path. + +Kept out of `internal_bypasser` so the HTTP layer can recognise an interstitial +without importing SeleniumBase: that module is imported lazily precisely because its +browser dependencies are optional, and external-bypasser setups run without them. +""" + +# Matched against lowercased text, so every entry must be lowercase. +CLOUDFLARE_INDICATORS = [ + "just a moment", + "verify you are human", + "verifying you are human", + "cloudflare.com/products/turnstile", +] + +DDOS_GUARD_INDICATORS = [ + "ddos-guard", + "ddos guard", + "checking your browser before accessing", + "complete the manual check to continue", + "could not verify your browser automatically", +] + +# Markers that exist only in raw markup: the bypassers scan rendered innerText, where +# a script src or a never appears. The title match is scoped to the tag on +# purpose - hosts word the rest of that sentence differently, and matching "checking +# your browser" as free text would trip on any page that merely discusses a challenge. +_RAW_HTML_MARKERS = ( + "<title>checking your browser", + "/cdn-cgi/challenge-platform", + "/.well-known/ddos-guard/", +) + +# An interstitial is a few KB of markup. Past that it is a real page that happens to +# mention a marker - a protected site links its own DDoS-Guard endpoints on every page. +MAX_CHALLENGE_HTML_CHARS = 64 * 1024 + + +def challenge_marker(html: str) -> str | None: + """Return the marker proving `html` is an unsolved challenge page, or None. + + Only meaningful for a response that already carries a challenge status: the + markers appear on protected sites' real pages too, so the status is what + separates "blocked" from "served". + """ + if not html or len(html) > MAX_CHALLENGE_HTML_CHARS: + return None + lowered = html.lower() + for marker in (*_RAW_HTML_MARKERS, *DDOS_GUARD_INDICATORS, *CLOUDFLARE_INDICATORS): + if marker in lowered: + return marker + return None diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index 429820e..647231f 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -27,6 +27,7 @@ from seleniumbase import cdp_driver from seleniumbase.undetected.cdp_driver.connection import ProtocolException from shelfmark.bypass import BypassCancelledError +from shelfmark.bypass.challenge import CLOUDFLARE_INDICATORS, DDOS_GUARD_INDICATORS from shelfmark.bypass.cookie_store import ( clear_cf_cookies, export_store, @@ -63,22 +64,6 @@ _IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS _BYPASS_CHILD_ENV = "SHELFMARK_INTERNAL_BYPASSER_CHILD" _PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0 -# Challenge detection indicators -CLOUDFLARE_INDICATORS = [ - "just a moment", - "verify you are human", - "verifying you are human", - "cloudflare.com/products/turnstile", -] - -DDOS_GUARD_INDICATORS = [ - "ddos-guard", - "ddos guard", - "checking your browser before accessing", - "complete the manual check to continue", - "could not verify your browser automatically", -] - class _DisplayState(TypedDict): ffmpeg: subprocess.Popen[bytes] | None diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index e010e35..2b5afce 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -11,6 +11,7 @@ import requests from tqdm import tqdm from shelfmark.bypass import BypassCancelledError, cookie_store +from shelfmark.bypass.challenge import challenge_marker from shelfmark.core.config import config as app_config from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int @@ -233,6 +234,22 @@ def _is_retryable_error(e: Exception) -> bool: _DEAD_MIRROR_CODES = (410, 451) +def _response_challenge_marker(response: requests.Response) -> str | None: + """The challenge marker in a response body, or None if it carries no challenge. + + Content type is checked first so a JSON or octet-stream error body is never + decoded just to be scanned; a missing header is scanned anyway, since an + interstitial served without one is still an interstitial. + """ + content_type = response.headers.get("Content-Type", "") + if content_type and "html" not in content_type.lower(): + return None + try: + return challenge_marker(response.text) + except UnicodeDecodeError, ValueError: + return None + + def _fatal_mirror_reason(e: Exception) -> str | None: """Return why ``e`` proves the mirror is unusable, or None if it may recover. @@ -455,6 +472,35 @@ def html_get_page( ) continue + # A 503 still serving a challenge is protection, not a busy origin. The + # handshake above has nothing left to echo back, and 503 is in + # RETRYABLE_CODES, so without this the request spends every attempt on + # the same wall: the bypasser is only ever reached from the 403 branch + # and the AA redirect rescues. Gate on the body, not the status, so a + # genuine overloaded-origin 503 keeps its retry path. + if response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE: + marker = _response_challenge_marker(response) + if marker and _bypass_handoff_allowed(): + if cookies: + # Challenged while presenting clearance means those cookies + # are dead; same reasoning as the 403 branch below. + logger.debug( + "503 challenge with cookies presented; purging: %s", current_url + ) + _purge_clearance(current_url) + logger.info( + "503 challenge detected (%s); switching to bypasser: %s", + marker, + current_url, + ) + return _run_bypasser(current_url) + if marker: + logger.debug( + "503 challenge (%s) but no bypasser handoff available: %s", + marker, + current_url, + ) + if is_aa_url and response.is_redirect: location = response.headers.get("Location", "") if not location: diff --git a/tests/download/test_http_challenge_503.py b/tests/download/test_http_challenge_503.py new file mode 100644 index 0000000..dbb544c --- /dev/null +++ b/tests/download/test_http_challenge_503.py @@ -0,0 +1,127 @@ +"""Tests for handing a 503 that carries a browser challenge to the bypasser.""" + +import requests + +_CHALLENGE_HTML = ( + "<html><head><title>Checking your browser before accessing z-lib.gd" + "" + "Please wait..." +) + + +class _FakeResponse: + """Minimal stand-in for requests.Response covering what html_get_page touches.""" + + def __init__( + self, + status_code: int, + *, + url: str = "https://z-lib.gd/md5/abc", + text: str = "", + cookies: dict[str, str] | None = None, + ) -> None: + self.status_code = status_code + self.url = url + self.text = text + self.cookies = cookies or {} + self.headers = {"Content-Type": "text/html;charset=utf-8"} + self.is_redirect = False + + def raise_for_status(self) -> None: + if self.status_code >= 400: + error = requests.exceptions.HTTPError(f"{self.status_code} Error") + error.response = self + raise error + + +def _neutralize_network(monkeypatch, http): + monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {}) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True) + monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False) + monkeypatch.setattr(http.time, "sleep", lambda _seconds: None) + + +def test_503_challenge_is_handed_to_the_bypasser(monkeypatch): + """The reissued-cookie 503 from #1233 reaches the bypasser instead of retrying.""" + import shelfmark.download.http as http + + _neutralize_network(monkeypatch, http) + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + attempts: list[dict[str, str]] = [] + bypassed: list[str] = [] + + def fake_get(_url: str, **kwargs): + attempts.append(dict(kwargs["cookies"])) + # Hit 1 issues the cookie; every later hit re-serves the challenge unchanged, + # which is what leaves the handshake with nothing to echo back. + if len(attempts) == 1: + return _FakeResponse(503, cookies={"bsrv": "1"}) + return _FakeResponse(503, text=_CHALLENGE_HTML, cookies={"bsrv": "1"}) + + def fake_bypass(url: str, _selector=None, _cancel_flag=None): + bypassed.append(url) + return "real page" + + monkeypatch.setattr(http.requests, "get", fake_get) + monkeypatch.setattr(http, "get_bypassed_page", fake_bypass) + + html = http.html_get_page("https://z-lib.gd/md5/abc", retry=10, success_delay=0) + + assert html == "real page" + assert bypassed == ["https://z-lib.gd/md5/abc"] + # The handshake still gets its echo; the challenge ends the loop on the second hit + # rather than burning all ten attempts. + assert attempts == [{}, {"bsrv": "1"}] + + +def test_plain_503_still_retries_without_bypassing(monkeypatch): + """An overloaded origin has no challenge marker, so its retry path is untouched.""" + import shelfmark.download.http as http + + _neutralize_network(monkeypatch, http) + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + attempts: list[dict[str, str]] = [] + bypassed: list[str] = [] + + def fake_get(_url: str, **kwargs): + attempts.append(dict(kwargs["cookies"])) + return _FakeResponse(503, text="Service Unavailable") + + monkeypatch.setattr(http.requests, "get", fake_get) + monkeypatch.setattr( + http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or "" + ) + + html = http.html_get_page("https://z-lib.gd/md5/abc", retry=3, success_delay=0) + + assert html == "" + assert bypassed == [] + assert attempts == [{}, {}, {}] + + +def test_503_challenge_respects_disabled_bypasser_fallback(monkeypatch): + """Best-effort fetches must not stall on a minutes-long solve.""" + import shelfmark.download.http as http + + _neutralize_network(monkeypatch, http) + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + bypassed: list[str] = [] + monkeypatch.setattr( + http.requests, + "get", + lambda _url, **_kwargs: _FakeResponse(503, text=_CHALLENGE_HTML), + ) + monkeypatch.setattr( + http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or "" + ) + + html = http.html_get_page( + "https://z-lib.gd/md5/abc", retry=1, success_delay=0, allow_bypasser_fallback=False + ) + + assert html == "" + assert bypassed == []