diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index e52f01d..9bb3552 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -434,6 +434,24 @@ def html_get_page( except Exception as e: status = _get_status_code(e) + # DDoS-Guard also answers stale clearance cookies with an endless `?check=1` + # redirect. TooManyRedirects carries no status, so the 403 rescue below never + # fires and every retry re-sends the dead cookies. + if ( + isinstance(e, requests.exceptions.TooManyRedirects) + and allow_bypasser_fallback + and _is_cf_bypass_enabled() + and not use_bypasser_now + ): + if not _is_using_external_bypasser(): + hostname = urlparse(current_url).hostname or "" + _get_internal_bypasser().clear_cf_cookies(hostname) + logger.info("Redirect loop detected; switching to bypasser: %s", current_url) + if status_callback: + status_callback("resolving", "Bypassing protection...") + use_bypasser_now = True + continue + # 403 = Cloudflare/DDoS-Guard protection if status == _HTTP_STATUS_FORBIDDEN: # If bypasser fallback is disabled, try mirrors instead diff --git a/tests/download/test_http_bypasser_fallbacks.py b/tests/download/test_http_bypasser_fallbacks.py index f159f7d..1716e8e 100644 --- a/tests/download/test_http_bypasser_fallbacks.py +++ b/tests/download/test_http_bypasser_fallbacks.py @@ -179,6 +179,65 @@ def test_challenged_search_switches_to_bypasser(monkeypatch): assert bypassed == ["https://annas-archive.gl/search?q=dune"] +def test_redirect_loop_purges_stale_cookies_and_switches_to_bypasser(monkeypatch): + """A stale clearance cookie turns the gate into a `?check=1` redirect loop. + + Guards the regression where TooManyRedirects carried no status code, so the + 403-only rescue never fired and every retry re-sent the dead cookie. + """ + import shelfmark.download.http as http + + stale = {"__ddg8_": "stale"} + cleared: list[str] = [] + + class _FakeInternalBypasser: + @staticmethod + def clear_cf_cookies(domain: str) -> None: + cleared.append(domain) + stale.clear() + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: False) + monkeypatch.setattr(http, "_get_internal_bypasser", lambda: _FakeInternalBypasser) + monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0) + monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: dict(stale)) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True) + monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True) + monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: True) + monkeypatch.setattr(http.time, "sleep", lambda _seconds: None) + + sent_cookies: list[dict[str, str]] = [] + + def check_redirect(url: str, **kwargs): + sent_cookies.append(kwargs["cookies"]) + response = _FakeResponse(302, url=url) + response.is_redirect = True + response.headers = {"Location": f"{url}&check=1"} + return response + + bypassed: list[str] = [] + monkeypatch.setattr(http.requests, "get", check_redirect) + monkeypatch.setattr( + http, + "get_bypassed_page", + lambda url, *_a, **_k: bypassed.append(url) or "results
", + ) + + html = http.html_get_page( + "https://annas-archive.gl/search?q=dune", + retry=10, + allow_bypasser_fallback=True, + success_delay=0, + ) + + assert html == "results
" + assert cleared == ["annas-archive.gl"] + assert len(bypassed) == 1 + # Escaped on the first exception, not retried with the dead cookie. + assert sent_cookies[0] == {"__ddg8_": "stale"} + + def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch): import shelfmark.download.http as http