diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index 488ff403..68ad07df 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -601,7 +601,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: f"{filters_query}" ) - html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False) + # AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating + # to another mirror only collects another 403, so let the bypasser solve it. + html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True) if not html: # Network/mirror exhaustion path bubbles up so API can notify clients msg = "Unable to reach download source. Network restricted or mirrors are blocked." @@ -657,7 +659,8 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR """ url = f"{network.get_aa_base_url()}/md5/{book_id}" selector = network.AAMirrorSelector() - html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False) + # Same challenge as search: the detail page is gated on every mirror, so bypass it. + html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True) if not html: msg = "Unable to reach download source. Network restricted or mirrors are blocked." @@ -885,6 +888,9 @@ def _parse_book_info_page( if fetch_download_count: try: summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}" + # Unlike search and the detail page above, this one stays off the bypasser: a + # download count is decoration on the details modal, not worth holding the + # modal open for a browser solve. If it is gated, drop it and move on. summary_response = downloader.html_get_page( summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False ) diff --git a/tests/download/test_http_bypasser_fallbacks.py b/tests/download/test_http_bypasser_fallbacks.py index 96ea131a..f159f7db 100644 --- a/tests/download/test_http_bypasser_fallbacks.py +++ b/tests/download/test_http_bypasser_fallbacks.py @@ -138,6 +138,47 @@ def test_html_get_page_returns_empty_on_bypass_cancellation(monkeypatch): assert html == "" +def test_challenged_search_switches_to_bypasser(monkeypatch): + """AA gates /search behind DDoS-Guard; the 403 must reach the bypasser, not a 503. + + Guards the regression where search passed allow_bypasser_fallback=False, so a + challenge on every mirror surfaced as "mirrors are blocked" with no solve attempted. + """ + import shelfmark.download.http as http + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0) + monkeypatch.setattr(http, "get_cf_cookies_for_domain", lambda _hostname: {}) + monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {}) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True) + monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False) + monkeypatch.setattr(http.time, "sleep", lambda _seconds: None) + + def ddos_guarded(url: str, **_kwargs): + error = requests.exceptions.HTTPError("forbidden") + error.response = _FakeResponse(403, url=url) + raise error + + bypassed: list[str] = [] + monkeypatch.setattr(http.requests, "get", ddos_guarded) + monkeypatch.setattr( + http, + "get_bypassed_page", + lambda url, *_a, **_k: bypassed.append(url) or "results
", + ) + + html = http.html_get_page( + "https://annas-archive.gl/search?q=dune", + retry=10, + allow_bypasser_fallback=True, + success_delay=0, + ) + + assert html == "results
" + assert bypassed == ["https://annas-archive.gl/search?q=dune"] + + def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch): import shelfmark.download.http as http diff --git a/tests/e2e/platform/README.md b/tests/e2e/platform/README.md index 058dee0d..010ffdb5 100644 --- a/tests/e2e/platform/README.md +++ b/tests/e2e/platform/README.md @@ -87,12 +87,14 @@ end-to-end (`docker compose up` + suite + teardown) and passes. | `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed | | *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ | -> **The bypasser is download-time, not search-time.** Running the stack revealed -> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a -> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser -> (internal Chrome or external FlareSolverr) only runs during a file *download* -> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean* -> CF-gated-search failure, while the **`full` profile exercises the real end-to-end +> **The bypasser now runs for search too.** AA search/detail used to be fetched with +> `allow_bypasser_fallback=False`, so a search behind Cloudflare returned 503 +> regardless of the bypasser — which left search dead when AA put DDoS-Guard in front +> of `/search`. Both now pass `allow_bypasser_fallback=True`, so a 403 switches +> straight to the bypasser rather than rotating to another mirror behind the same +> gate. The bypasser profiles therefore assert that a gated search *recovers* when the +> external bypasser is available and still fails cleanly when it is off, while the +> **`full` profile exercises the real end-to-end > CF solve**: AA search/detail are reachable, but the AA slow-download link points > at the gate, so downloading Moby-Dick forces the in-image headless Chromium to > detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file — diff --git a/tests/e2e/platform/suite/test_cluster_bypasser.py b/tests/e2e/platform/suite/test_cluster_bypasser.py index 87cd2182..e1e3b5b1 100644 --- a/tests/e2e/platform/suite/test_cluster_bypasser.py +++ b/tests/e2e/platform/suite/test_cluster_bypasser.py @@ -1,20 +1,18 @@ -"""Cluster 1 — Cloudflare bypasser wiring + clean-failure behavior. +"""Cluster 1 — Cloudflare bypasser wiring + gated-search behavior. -Reality discovered by running the stack: shelfmark's AA *search* and *detail* -fetches use ``html_get_page(allow_bypasser_fallback=False)``, so a search behind a -Cloudflare gate returns 503 **regardless** of the bypasser. The bypasser (internal -Chrome or external FlareSolverr) is a *download-time* mechanism -(``html_get_page(use_bypasser=True)``); it never runs for search. +AA search/detail used to be fetched with ``allow_bypasser_fallback=False``, so a +search behind a Cloudflare gate returned 503 no matter how the bypasser was +configured — which left search dead when AA put DDoS-Guard in front of ``/search``. +Both now pass ``allow_bypasser_fallback=True``: a 403 switches straight to the +bypasser instead of rotating to another mirror behind the same gate. -So these tests assert what is actually true and host-observable: - * the external bypasser is configured from env, and - * a CF-gated search fails *cleanly* (a 503 the client can act on, not a hang or - a crash) — both with the bypasser on (it isn't used for search) and off. +So these tests assert: + * the external bypasser is configured from env, + * a CF-gated search *recovers* once the external bypasser is available, and + * with the bypasser off it still fails *cleanly* (a 503 the client can act on, + not a hang or a crash) — the negative control that the gate is not ignored. -Exercising shelfmark's *use* of the bypasser end-to-end (a real CF solve during a -download) needs the AA slow-download HTML flow mocked — see the README roadmap. -The bypass *mechanism* itself is verified to work: the mock FlareSolverr solves -the gate (manually confirmed; see README). Guards: #284 #226 #202 #1030 #410 #369. +Guards: #284 #226 #202 #1030 #410 #369. """ from __future__ import annotations @@ -70,10 +68,26 @@ def test_external_bypasser_is_configured(client) -> None: @pytest.mark.profiles("bypasser-external") -def test_cf_gated_search_fails_cleanly_with_external_bypasser(client) -> None: - """Even with the external bypasser configured, a CF-gated *search* yields no - releases (the bypasser is download-time) — but it must fail cleanly.""" - assert _cf_gated_search_has_no_releases(client) +def test_cf_gated_search_recovers_via_external_bypasser(client) -> None: + """A CF-gated search is solved through the bypasser instead of returning 503. + + The mock FlareSolverr refetches with the clearance cookie, which the cloudflare + role then passes through to the AA origin — so the 403 that used to end the + search now turns into real results. + """ + resp = client.get( + "/api/releases", + params={"source": "direct_download", "query": "Mistborn"}, + timeout=120, + ) + assert resp.status_code == 200, ( + f"CF-gated search did not recover through the external bypasser: " + f"{resp.status_code} {resp.text[:200]}" + ) + assert client.releases_from(resp), ( + "external bypasser was configured but the gated search returned no releases — " + "the 403 did not switch search over to the bypasser" + ) @pytest.mark.profiles("bypasser-disabled")