"""A challenge page is a failed solve, whatever verdict the solver reports on itself. Regression for #1292. FlareSolverr answers "Challenge solved!" for anything it does not recognise as a Cloudflare challenge, and DDoS-Guard's manual CAPTCHA page is one such thing. The external bypasser logged a warning that the solve had not cleared the protection and then returned the page as a success anyway, which had three consequences: the retry-and-rotate loop that could still have reached a working mirror was never entered, the CAPTCHA page's own __ddg cookies were filed as that host's clearance, and the user was told to go and check a bypasser that was working perfectly. """ import pytest from shelfmark.bypass import ChallengeNotSolvedError # Verbatim from the annas-archive.pk page in #1292, trimmed to the markers. This is the # *manual* CAPTCHA - "could not verify your browser automatically" - not the ~900 byte # JS interstitial that a browser clears on its own. DDOS_GUARD_CAPTCHA = ( 'DDOS-GUARD' '' '' '

Checking your browser before ' 'accessing annas-archive.pk

Sorry, we could not verify your ' "browser automatically. Complete the manual check to continue

" '
' ) class _FakeResponse: def __init__(self, payload: dict) -> None: self._payload = payload def raise_for_status(self) -> None: return None def json(self) -> dict: return self._payload def _stub_solution(monkeypatch, external_bypasser, solution: dict) -> None: """Answer every bypass with `solution`, with config and SSL stubbed out.""" def fake_get(key, default=""): values = { "EXT_BYPASSER_URL": "https://bypass.example", "EXT_BYPASSER_PATH": "/v1", "EXT_BYPASSER_TIMEOUT": 60000, } return values.get(key, default) monkeypatch.setattr(external_bypasser.config, "get", fake_get) monkeypatch.setattr( external_bypasser.requests, "post", # "Challenge solved!" is the solver's verdict; the page is the evidence. lambda *_a, **_k: _FakeResponse( {"status": "ok", "message": "Challenge solved!", "solution": solution} ), ) monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False) def test_a_captcha_page_is_reported_as_unsolved_not_returned(monkeypatch): import shelfmark.bypass.external_bypasser as external_bypasser _stub_solution(monkeypatch, external_bypasser, {"response": DDOS_GUARD_CAPTCHA}) with pytest.raises(ChallengeNotSolvedError) as excinfo: external_bypasser._fetch_via_bypasser("https://annas-archive.pk/search?q=dune") # The marker travels with the failure so the user-facing message can name it. assert str(excinfo.value) == "/.well-known/ddos-guard/" def test_cookies_from_a_captcha_page_are_never_filed_as_clearance(monkeypatch): """They belong to an unsolved check, so replaying them only re-arms the gate.""" import shelfmark.bypass.cookie_store as cookie_store import shelfmark.bypass.external_bypasser as external_bypasser monkeypatch.setattr(cookie_store, "_cf_cookies", {}) monkeypatch.setattr(cookie_store, "_cf_user_agents", {}) _stub_solution( monkeypatch, external_bypasser, { "response": DDOS_GUARD_CAPTCHA, "userAgent": "Mozilla/5.0 (solver)", "cookies": [{"name": "__ddg1_", "value": "from-a-captcha"}], }, ) with pytest.raises(ChallengeNotSolvedError): external_bypasser._fetch_via_bypasser("https://annas-archive.pk/search?q=dune") assert cookie_store.get_cf_cookies_for_domain("annas-archive.pk") == {} assert cookie_store.get_cf_user_agent_for_domain("annas-archive.pk") is None class _FakeSelector: """Two mirrors, rotated on demand - each is its own DDoS-Guard host.""" def __init__(self) -> None: self.current_base = "https://mirror-one.example" self.rotate_calls = 0 def rewrite(self, url: str) -> str: return url.replace("https://orig.example", self.current_base, 1) def next_mirror_or_rotate_dns(self) -> tuple[str | None, str]: self.rotate_calls += 1 self.current_base = "https://mirror-two.example" return self.current_base, "mirror" def _no_sleeping(monkeypatch, external_bypasser) -> None: monkeypatch.setattr(external_bypasser, "_sleep_with_cancellation", lambda _seconds, _flag: None) def test_an_unsolved_challenge_rotates_to_the_next_mirror(monkeypatch): """The recovery the old code skipped by calling the CAPTCHA page a success.""" import shelfmark.bypass.external_bypasser as external_bypasser _no_sleeping(monkeypatch, external_bypasser) fetched: list[str] = [] def fake_fetch(url: str) -> str | None: fetched.append(url) if "mirror-one" in url: raise ChallengeNotSolvedError("/.well-known/ddos-guard/") return "real page" monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", fake_fetch) selector = _FakeSelector() result = external_bypasser.get_bypassed_page("https://orig.example/search", selector=selector) assert result == "real page" assert fetched == [ "https://mirror-one.example/search", "https://mirror-two.example/search", ] assert selector.rotate_calls == 1 def test_every_attempt_challenged_blames_the_host_not_the_bypasser(monkeypatch): import shelfmark.bypass.external_bypasser as external_bypasser _no_sleeping(monkeypatch, external_bypasser) def always_challenged(_url: str) -> str | None: raise ChallengeNotSolvedError("/.well-known/ddos-guard/") monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", always_challenged) with pytest.raises(ChallengeNotSolvedError) as excinfo: external_bypasser.get_bypassed_page("https://orig.example/search", selector=_FakeSelector()) message = str(excinfo.value) assert "manual CAPTCHA" in message assert "the bypasser itself is working" in message def test_an_unreachable_bypasser_still_reports_as_such(monkeypatch): """The other cause must stay distinguishable: None, not an unsolved challenge.""" import shelfmark.bypass.external_bypasser as external_bypasser _no_sleeping(monkeypatch, external_bypasser) monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", lambda _url: None) assert ( external_bypasser.get_bypassed_page("https://orig.example/search", selector=_FakeSelector()) is None ) def test_html_get_page_surfaces_the_host_as_the_cause(monkeypatch): """The message the user actually reads must not send them to fix FlareSolverr. `_run_bypasser`'s generic handler says "the protection bypasser failed", and the search layer's give-up used to add "check that the bypasser is reachable and working" - which is what #1292 spent its investigation doing. """ import shelfmark.download.http as http import shelfmark.download.network as network monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True) def challenged(*_args, **_kwargs): msg = "the site kept answering with a protection challenge - manual CAPTCHA" raise ChallengeNotSolvedError(msg) monkeypatch.setattr(http, "get_bypassed_page", challenged) statuses: list[tuple[str, str | None]] = [] selector = network.AAMirrorSelector() html = http.html_get_page( "https://annas-archive.pk/search?q=dune", retry=1, selector=selector, status_callback=lambda stage, detail: statuses.append((stage, detail)), use_bypasser=True, success_delay=0, ) assert html == "" assert selector.last_failure is not None assert "manual CAPTCHA" in selector.last_failure assert "reachable" not in selector.last_failure assert ("error", "the site kept answering with a protection challenge - manual CAPTCHA") in ( statuses )