Let the bypasser solve bot challenges on Anna's Archive search (#1198)

Anna's Archive put a DDoS-Guard JS challenge in front of /search: the
homepage still returns 200, but /search and /md5/<id> answer 403 on
every mirror (.gl, .pk, .gd all confirmed). Search fetched both with
allow_bypasser_fallback=False, which rotates mirrors on a 403 instead of
invoking the bypasser, so it walked the whole mirror list, exhausted it,
and surfaced "Unable to reach download source. Network restricted or
mirrors are blocked." as a 503 on every query. Adding mirrors could not
help — they sit behind the same gate — and neither could USE_CF_BYPASS,
since search never reached that branch.
Fetch search and the detail page with allow_bypasser_fallback=True so a
403 hands over to the bypasser, which already detects this challenge
(DDOS_GUARD_INDICATORS matches the live page). Echoing the __ddg cookies
back does not clear it; it needs real JS execution. The download-count
fetch keeps allow_bypasser_fallback=False: it is decoration on the
details modal and not worth holding the modal open for a browser solve.
Fixes #1196

shelfmark/release_sources/direct_download.py

@@ -601,7 +601,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
         f"{filters_query}"
     )
 
-    html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
+    # AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating
+    # to another mirror only collects another 403, so let the bypasser solve it.
+    html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True)
     if not html:
         # Network/mirror exhaustion path bubbles up so API can notify clients
         msg = "Unable to reach download source. Network restricted or mirrors are blocked."
@@ -657,7 +659,8 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR
     """
     url = f"{network.get_aa_base_url()}/md5/{book_id}"
     selector = network.AAMirrorSelector()
-    html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
+    # Same challenge as search: the detail page is gated on every mirror, so bypass it.
+    html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True)
 
     if not html:
         msg = "Unable to reach download source. Network restricted or mirrors are blocked."
@@ -885,6 +888,9 @@ def _parse_book_info_page(
     if fetch_download_count:
         try:
             summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}"
+            # Unlike search and the detail page above, this one stays off the bypasser: a
+            # download count is decoration on the details modal, not worth holding the
+            # modal open for a browser solve. If it is gated, drop it and move on.
             summary_response = downloader.html_get_page(
                 summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False
             )

tests/download/test_http_bypasser_fallbacks.py

@@ -138,6 +138,47 @@ def test_html_get_page_returns_empty_on_bypass_cancellation(monkeypatch):
     assert html == ""
 
 
+def test_challenged_search_switches_to_bypasser(monkeypatch):
+    """AA gates /search behind DDoS-Guard; the 403 must reach the bypasser, not a 503.
+
+    Guards the regression where search passed allow_bypasser_fallback=False, so a
+    challenge on every mirror surfaced as "mirrors are blocked" with no solve attempted.
+    """
+    import shelfmark.download.http as http
+
+    monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
+    monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
+    monkeypatch.setattr(http, "get_cf_cookies_for_domain", lambda _hostname: {})
+    monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
+    monkeypatch.setattr(http, "get_proxies", lambda _url: {})
+    monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
+    monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
+    monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
+
+    def ddos_guarded(url: str, **_kwargs):
+        error = requests.exceptions.HTTPError("forbidden")
+        error.response = _FakeResponse(403, url=url)
+        raise error
+
+    bypassed: list[str] = []
+    monkeypatch.setattr(http.requests, "get", ddos_guarded)
+    monkeypatch.setattr(
+        http,
+        "get_bypassed_page",
+        lambda url, *_a, **_k: bypassed.append(url) or "<table>results</table>",
+    )
+
+    html = http.html_get_page(
+        "https://annas-archive.gl/search?q=dune",
+        retry=10,
+        allow_bypasser_fallback=True,
+        success_delay=0,
+    )
+
+    assert html == "<table>results</table>"
+    assert bypassed == ["https://annas-archive.gl/search?q=dune"]
+
+
 def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch):
     import shelfmark.download.http as http
 

tests/e2e/platform/README.md

@@ -87,12 +87,14 @@ end-to-end (`docker compose up` + suite + teardown) and passes.
 | `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed |
 | *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ |
 
-> **The bypasser is download-time, not search-time.** Running the stack revealed
-> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a
-> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser
-> (internal Chrome or external FlareSolverr) only runs during a file *download*
-> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean*
-> CF-gated-search failure, while the **`full` profile exercises the real end-to-end
+> **The bypasser now runs for search too.** AA search/detail used to be fetched with
+> `allow_bypasser_fallback=False`, so a search behind Cloudflare returned 503
+> regardless of the bypasser — which left search dead when AA put DDoS-Guard in front
+> of `/search`. Both now pass `allow_bypasser_fallback=True`, so a 403 switches
+> straight to the bypasser rather than rotating to another mirror behind the same
+> gate. The bypasser profiles therefore assert that a gated search *recovers* when the
+> external bypasser is available and still fails cleanly when it is off, while the
+> **`full` profile exercises the real end-to-end
 > CF solve**: AA search/detail are reachable, but the AA slow-download link points
 > at the gate, so downloading Moby-Dick forces the in-image headless Chromium to
 > detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file —

tests/e2e/platform/suite/test_cluster_bypasser.py

@@ -1,20 +1,18 @@
-"""Cluster 1 — Cloudflare bypasser wiring + clean-failure behavior.
+"""Cluster 1 — Cloudflare bypasser wiring + gated-search behavior.
 
-Reality discovered by running the stack: shelfmark's AA *search* and *detail*
-fetches use ``html_get_page(allow_bypasser_fallback=False)``, so a search behind a
-Cloudflare gate returns 503 **regardless** of the bypasser. The bypasser (internal
-Chrome or external FlareSolverr) is a *download-time* mechanism
-(``html_get_page(use_bypasser=True)``); it never runs for search.
+AA search/detail used to be fetched with ``allow_bypasser_fallback=False``, so a
+search behind a Cloudflare gate returned 503 no matter how the bypasser was
+configured — which left search dead when AA put DDoS-Guard in front of ``/search``.
+Both now pass ``allow_bypasser_fallback=True``: a 403 switches straight to the
+bypasser instead of rotating to another mirror behind the same gate.
 
-So these tests assert what is actually true and host-observable:
-  * the external bypasser is configured from env, and
-  * a CF-gated search fails *cleanly* (a 503 the client can act on, not a hang or
-    a crash) — both with the bypasser on (it isn't used for search) and off.
+So these tests assert:
+  * the external bypasser is configured from env,
+  * a CF-gated search *recovers* once the external bypasser is available, and
+  * with the bypasser off it still fails *cleanly* (a 503 the client can act on,
+    not a hang or a crash) — the negative control that the gate is not ignored.
 
-Exercising shelfmark's *use* of the bypasser end-to-end (a real CF solve during a
-download) needs the AA slow-download HTML flow mocked — see the README roadmap.
-The bypass *mechanism* itself is verified to work: the mock FlareSolverr solves
-the gate (manually confirmed; see README). Guards: #284 #226 #202 #1030 #410 #369.
+Guards: #284 #226 #202 #1030 #410 #369.
 """
 
 from __future__ import annotations
@@ -70,10 +68,26 @@ def test_external_bypasser_is_configured(client) -> None:
 
 
 @pytest.mark.profiles("bypasser-external")
-def test_cf_gated_search_fails_cleanly_with_external_bypasser(client) -> None:
-    """Even with the external bypasser configured, a CF-gated *search* yields no
-    releases (the bypasser is download-time) — but it must fail cleanly."""
-    assert _cf_gated_search_has_no_releases(client)
+def test_cf_gated_search_recovers_via_external_bypasser(client) -> None:
+    """A CF-gated search is solved through the bypasser instead of returning 503.
+
+    The mock FlareSolverr refetches with the clearance cookie, which the cloudflare
+    role then passes through to the AA origin — so the 403 that used to end the
+    search now turns into real results.
+    """
+    resp = client.get(
+        "/api/releases",
+        params={"source": "direct_download", "query": "Mistborn"},
+        timeout=120,
+    )
+    assert resp.status_code == 200, (
+        f"CF-gated search did not recover through the external bypasser: "
+        f"{resp.status_code} {resp.text[:200]}"
+    )
+    assert client.releases_from(resp), (
+        "external bypasser was configured but the gated search returned no releases — "
+        "the 403 did not switch search over to the bypasser"
+    )
 
 
 @pytest.mark.profiles("bypasser-disabled")