mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 20:10:28 +01:00
CaliBrain
651096ed7b
Direct download was unusable behind an external bypasser (FlareSolverr /
Byparr): every request paid a 403 plus a full solve, and a search that
never ran was reported to the user as "No books found".
Clearance was discarded on the external path. get_cf_cookies_for_domain
and get_cf_user_agent_for_domain returned {} / None whenever
USING_EXTERNAL_BYPASSER was set, and _fetch_via_bypasser read only
solution.response - dropping solution.cookies and solution.userAgent,
which FlareSolverr-compatible services do return. A solve therefore
cleared the one request that paid for it and nothing else, and a file
download - which the solver cannot proxy, being binary - presented no
clearance at all. Diagnosed from a v1.3.9 debug bundle: ~35s in the
bypasser per search, on every search.
- Move the cookie jar out of internal_bypasser into bypass/cookie_store.
internal_bypasser imports seleniumbase at module scope, which is the
dependency an external-bypasser deployment is entitled not to have, so
it cannot host a store the external path depends on.
- Harvest solution.cookies and solution.userAgent after a successful
solve. The existing filtering applies unchanged, so the per-check
__ddg8_/__ddg9_/__ddg10_ trio is still dropped and the external path
cannot reintroduce the ?check=1 loop fixed in ebb833a. The UA matters
as much as the cookies: Cloudflare ties cf_clearance to the UA that
solved the challenge.
- Read cookie fields from either shape - CDP objects or JSON mappings.
Both use the same field names, expires included.
- Point http.py's getters and _purge_clearance at the shared store, so
either bypasser fills and drains the same jar.
- Give the Docker helper-subprocess handoff explicit export_store /
import_store rather than reaching into module globals.
An unsolved challenge was also indistinguishable from an empty result.
_looks_like_aa_page() counted the challenge markers as "recognisably
AA", so _fetch_search_table handed a DDoS-Guard interstitial back as a
legitimate no-table response and the user was told their query found
nothing when the search never ran. Split challenge detection out and
raise SearchUnavailableError with the reason instead. The mirror is
still not quarantined - every mirror shares the same protection, so it
is not the mirror's fault.
Verified: 2531 unit tests pass; ruff, basedpyright and vulture clean;
e2e bypasser-external profile passes (5). Its mock FlareSolverr already
returned cookies and userAgent from /v1 - the contract was there,
shelfmark was not reading it.
Refs #1220. Deliberately not "Fixes": this removes the re-solve and
makes a failed solve legible, but if Byparr genuinely cannot clear AA's
current DDoS-Guard, the reporter now gets that as an error rather than a
silent "no books found". The download path may swallow interstitials the
same way; not audited here.
298 lines
10 KiB
Python
298 lines
10 KiB
Python
"""Tests for the external bypasser flow."""
|
|
|
|
import pytest
|
|
|
|
|
|
class _FakeResponse:
|
|
def __init__(self, payload: dict) -> None:
|
|
self._payload = payload
|
|
|
|
def raise_for_status(self) -> None:
|
|
return None
|
|
|
|
def json(self) -> dict:
|
|
return self._payload
|
|
|
|
|
|
def test_fetch_via_bypasser_posts_expected_payload_and_uses_ssl_verify(monkeypatch):
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
calls: list[dict] = []
|
|
|
|
def fake_get(key, default=""):
|
|
values = {
|
|
"EXT_BYPASSER_URL": "https://bypass.example",
|
|
"EXT_BYPASSER_PATH": "/v1",
|
|
"EXT_BYPASSER_TIMEOUT": 60000,
|
|
}
|
|
return values.get(key, default)
|
|
|
|
def fake_post(url: str, **kwargs):
|
|
calls.append({"url": url, **kwargs})
|
|
return _FakeResponse(
|
|
{
|
|
"status": "ok",
|
|
"message": "done",
|
|
"solution": {"response": "<html>ok</html>"},
|
|
}
|
|
)
|
|
|
|
monkeypatch.setattr(external_bypasser.config, "get", fake_get)
|
|
monkeypatch.setattr(external_bypasser.requests, "post", fake_post)
|
|
monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False)
|
|
|
|
assert external_bypasser._fetch_via_bypasser("https://example.com/book") == "<html>ok</html>"
|
|
assert calls == [
|
|
{
|
|
"url": "https://bypass.example/v1",
|
|
"headers": {"Content-Type": "application/json"},
|
|
"json": {
|
|
"cmd": "request.get",
|
|
"url": "https://example.com/book",
|
|
"maxTimeout": 60000,
|
|
},
|
|
"timeout": (10, 75.0),
|
|
"verify": False,
|
|
}
|
|
]
|
|
|
|
|
|
def _stub_solution(monkeypatch, external_bypasser, solution: dict) -> None:
|
|
"""Answer one bypass with `solution`, with config and SSL stubbed out."""
|
|
|
|
def fake_get(key, default=""):
|
|
values = {
|
|
"EXT_BYPASSER_URL": "https://bypass.example",
|
|
"EXT_BYPASSER_PATH": "/v1",
|
|
"EXT_BYPASSER_TIMEOUT": 60000,
|
|
}
|
|
return values.get(key, default)
|
|
|
|
monkeypatch.setattr(external_bypasser.config, "get", fake_get)
|
|
monkeypatch.setattr(
|
|
external_bypasser.requests,
|
|
"post",
|
|
lambda *_a, **_k: _FakeResponse({"status": "ok", "solution": solution}),
|
|
)
|
|
monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False)
|
|
|
|
|
|
def test_solved_clearance_is_stored_for_reuse(monkeypatch):
|
|
"""A solve costs tens of seconds of real browser; its clearance must be kept.
|
|
|
|
Without this every request paid a 403 plus a full solve, and the file download -
|
|
which the solver cannot proxy - presented no clearance at all.
|
|
"""
|
|
import shelfmark.bypass.cookie_store as cookie_store
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
|
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
|
_stub_solution(
|
|
monkeypatch,
|
|
external_bypasser,
|
|
{
|
|
"response": "<html>ok</html>",
|
|
"userAgent": "Mozilla/5.0 (solver)",
|
|
"cookies": [
|
|
{"name": "__ddg1_", "value": "clearance", "domain": ".annas-archive.gl"},
|
|
{"name": "__ddg2_", "value": "c2", "domain": ".annas-archive.gl"},
|
|
# Per-check cookies: kept out of the store, same as the internal path.
|
|
{"name": "__ddg9_", "value": "203.0.113.7", "domain": ".annas-archive.gl"},
|
|
],
|
|
},
|
|
)
|
|
|
|
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
|
|
|
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {
|
|
"__ddg1_": "clearance",
|
|
"__ddg2_": "c2",
|
|
}
|
|
# Cloudflare ties clearance to the solving UA, so replaying one without the other fails.
|
|
assert cookie_store.get_cf_user_agent_for_domain("annas-archive.gl") == "Mozilla/5.0 (solver)"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "shape"),
|
|
[
|
|
# Byparr drives Playwright/camoufox, whose cookies spell it "expires".
|
|
("expires", "playwright"),
|
|
# FlareSolverr assigns driver.get_cookies() - the WebDriver cookie object,
|
|
# which spells it "expiry". Reading only "expires" made every FlareSolverr
|
|
# cookie immortal, so dead clearance was replayed forever.
|
|
("expiry", "webdriver"),
|
|
],
|
|
)
|
|
def test_expired_solution_cookie_is_not_replayed(monkeypatch, field, shape):
|
|
import time
|
|
|
|
import shelfmark.bypass.cookie_store as cookie_store
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
|
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
|
_stub_solution(
|
|
monkeypatch,
|
|
external_bypasser,
|
|
{
|
|
"response": "<html>ok</html>",
|
|
"cookies": [{"name": "__ddg1_", "value": "dead", field: int(time.time()) - 60}],
|
|
},
|
|
)
|
|
|
|
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
|
|
|
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}, (
|
|
f"a dead {shape} cookie was kept for replay"
|
|
)
|
|
|
|
|
|
def test_solution_cookie_expiry_is_coerced_not_trusted(monkeypatch):
|
|
"""The solver is not ours; a stringified expiry must be read, not raised on."""
|
|
import time
|
|
|
|
import shelfmark.bypass.cookie_store as cookie_store
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
|
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
|
_stub_solution(
|
|
monkeypatch,
|
|
external_bypasser,
|
|
{
|
|
"response": "<html>ok</html>",
|
|
"cookies": [
|
|
{"name": "__ddg1_", "value": "live", "expires": str(int(time.time()) + 3600)},
|
|
{"name": "__ddg2_", "value": "dead", "expires": str(int(time.time()) - 60)},
|
|
],
|
|
},
|
|
)
|
|
|
|
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
|
|
|
assert result == "<html>ok</html>"
|
|
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
|
|
|
|
|
def test_storing_clearance_can_never_discard_the_solved_page(monkeypatch):
|
|
"""A solve costs ~30s; a surprise in the cookie shape must not throw it away.
|
|
|
|
The store call sits inside the request try/except, whose handler returns None -
|
|
so without its own guard a raising store turned a good page into a failed fetch
|
|
and sent the caller round for up to MAX_RETRY more solves.
|
|
"""
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
_stub_solution(
|
|
monkeypatch,
|
|
external_bypasser,
|
|
{"response": "<html>ok</html>", "cookies": [{"name": "__ddg1_", "value": "v"}]},
|
|
)
|
|
|
|
def boom(*_args, **_kwargs):
|
|
raise TypeError("unexpected cookie shape")
|
|
|
|
monkeypatch.setattr(external_bypasser, "store_extracted_cookies", boom)
|
|
|
|
assert (
|
|
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
|
== "<html>ok</html>"
|
|
)
|
|
|
|
|
|
def test_solution_without_cookies_is_still_returned(monkeypatch):
|
|
"""A solver that returns no cookie list must not break the page fetch."""
|
|
import shelfmark.bypass.cookie_store as cookie_store
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
|
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
|
_stub_solution(monkeypatch, external_bypasser, {"response": "<html>ok</html>"})
|
|
|
|
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
|
|
|
assert result == "<html>ok</html>"
|
|
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
|
|
|
|
|
def test_get_bypassed_page_retries_and_rotates_selector_between_attempts(monkeypatch):
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
class FakeRng:
|
|
def random(self) -> float:
|
|
return 0.0
|
|
|
|
class FakeSelector:
|
|
def __init__(self) -> None:
|
|
self.current_base = "https://mirror-one.example"
|
|
self.rewrite_calls: list[str] = []
|
|
self.rotate_calls = 0
|
|
|
|
def rewrite(self, url: str) -> str:
|
|
self.rewrite_calls.append(url)
|
|
return url.replace("https://orig.example", self.current_base, 1)
|
|
|
|
def next_mirror_or_rotate_dns(self) -> tuple[str | None, str]:
|
|
self.rotate_calls += 1
|
|
self.current_base = "https://mirror-two.example"
|
|
return self.current_base, "mirror"
|
|
|
|
fetch_calls: list[str] = []
|
|
sleeps: list[float] = []
|
|
responses = [None, "<html>ok</html>"]
|
|
|
|
def fake_fetch(url: str) -> str | None:
|
|
fetch_calls.append(url)
|
|
return responses.pop(0)
|
|
|
|
monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", fake_fetch)
|
|
monkeypatch.setattr(
|
|
external_bypasser, "_sleep_with_cancellation", lambda seconds, _flag: sleeps.append(seconds)
|
|
)
|
|
monkeypatch.setattr(external_bypasser, "_RNG", FakeRng())
|
|
|
|
selector = FakeSelector()
|
|
result = external_bypasser.get_bypassed_page("https://orig.example/book", selector=selector)
|
|
|
|
assert result == "<html>ok</html>"
|
|
assert fetch_calls == [
|
|
"https://mirror-one.example/book",
|
|
"https://mirror-two.example/book",
|
|
]
|
|
assert selector.rotate_calls == 1
|
|
assert sleeps == [1.0]
|
|
|
|
|
|
def _stub_ext_bypasser_timeout(monkeypatch, external_bypasser, value: int) -> None:
|
|
"""Override only EXT_BYPASSER_TIMEOUT on the shared config singleton."""
|
|
real_get = external_bypasser.config.get
|
|
monkeypatch.setattr(
|
|
external_bypasser.config,
|
|
"get",
|
|
lambda key, default="": value if key == "EXT_BYPASSER_TIMEOUT" else real_get(key, default),
|
|
)
|
|
|
|
|
|
def test_max_duration_seconds_covers_every_attempt_and_backoff(monkeypatch):
|
|
"""The declared budget must not undercut what get_bypassed_page() can actually take."""
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
_stub_ext_bypasser_timeout(monkeypatch, external_bypasser, 60000)
|
|
|
|
budget = external_bypasser.max_duration_seconds()
|
|
|
|
# 5 attempts at min(60 + 15, 120) = 75s, plus the 1+2+4+8 backoff and its jitter.
|
|
assert budget == 5 * 75.0 + (1 + 1) + (2 + 1) + (4 + 1) + (8 + 1)
|
|
|
|
|
|
def test_max_duration_seconds_respects_the_read_timeout_ceiling(monkeypatch):
|
|
import shelfmark.bypass.external_bypasser as external_bypasser
|
|
|
|
_stub_ext_bypasser_timeout(monkeypatch, external_bypasser, 300000)
|
|
|
|
budget = external_bypasser.max_duration_seconds()
|
|
|
|
# 300s + 15s buffer is clamped to MAX_READ_TIMEOUT, not used raw.
|
|
assert budget == 5 * external_bypasser.MAX_READ_TIMEOUT + 19.0
|