mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 20:20:20 +01:00
269 lines
9.5 KiB
Python
269 lines
9.5 KiB
Python
"""How long the internal bypasser is allowed to spend, and on what.
|
|
|
|
Issue #1276: MAX_RETRY drove *both* the outer page-load loop and the per-page method
|
|
loop, so the default of 10 meant ~40 solve attempts on one browser. That overran the
|
|
worker deadline, and the failure reached the user as `RuntimeError: TimeoutError` - a
|
|
message that says nothing about a protection challenge and sent people looking at their
|
|
reverse proxy instead.
|
|
|
|
Also covered: the undisturbed window a passive challenge gets before anything touches the
|
|
page. Anna's Archive's DDoS-Guard check has no click target and clears itself; going
|
|
straight to the click/reload methods meant the one thing that solves it was never tried.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def bypass(monkeypatch):
|
|
"""internal_bypasser with sleeps and jitter removed."""
|
|
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
|
|
|
async def _no_sleep(_seconds) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(internal_bypasser.asyncio, "sleep", _no_sleep)
|
|
monkeypatch.setattr(internal_bypasser._RNG, "uniform", lambda _a, _b: 0)
|
|
return internal_bypasser
|
|
|
|
|
|
def _recording_methods(calls: list[str], count: int = 4):
|
|
def _make(name: str):
|
|
async def _method(_page) -> bool:
|
|
calls.append(name)
|
|
return False
|
|
|
|
_method.__name__ = name
|
|
return _method
|
|
|
|
return [_make(f"m{i}") for i in range(count)]
|
|
|
|
|
|
def _stub_page_state(monkeypatch, bypass, *, bypassed=False, challenge="ddos_guard"):
|
|
async def _is_bypassed(*_args, **_kwargs) -> bool:
|
|
return bypassed
|
|
|
|
async def _detect(*_args, **_kwargs) -> str:
|
|
return challenge
|
|
|
|
monkeypatch.setattr(bypass, "_is_bypassed", _is_bypassed)
|
|
monkeypatch.setattr(bypass, "_detect_challenge_type", _detect)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# The method loop must not read MAX_RETRY
|
|
# --------------------------------------------------------------------------- #
|
|
def test_method_loop_budget_is_independent_of_max_retry(monkeypatch, bypass):
|
|
"""MAX_RETRY is the outer page-load retry; reading it here squared the budget.
|
|
|
|
Exercised against a challenge whose *type* keeps changing, because that is the case
|
|
where max_retries is what bounds the loop: the stuck-challenge guard only fires on a
|
|
run of the same type, so with a stable challenge it hid the real budget entirely.
|
|
"""
|
|
monkeypatch.setattr(type(bypass.app_config), "MAX_RETRY", 50, raising=False)
|
|
|
|
types = iter(["ddos_guard", "cloudflare"] * 100)
|
|
|
|
async def _alternating(*_args, **_kwargs) -> str:
|
|
return next(types)
|
|
|
|
calls: list[str] = []
|
|
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
|
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _never_passes)
|
|
_stub_page_state(monkeypatch, bypass)
|
|
monkeypatch.setattr(bypass, "_detect_challenge_type", _alternating)
|
|
|
|
assert asyncio.run(bypass._bypass(object())) is False
|
|
assert len(calls) == bypass._BYPASS_METHOD_ATTEMPTS
|
|
assert len(calls) < 50, "MAX_RETRY must not reach the method loop"
|
|
|
|
|
|
def test_method_attempt_budget_is_reachable(bypass):
|
|
"""The number reported as `attempt N/X` must be a number the loop can reach.
|
|
|
|
It used to be MAX_RETRY (10) while the stuck-challenge guard capped the loop at 5,
|
|
so logs showed `4/10` and stopped, which reads like six lost attempts.
|
|
"""
|
|
assert bypass._BYPASS_METHOD_ATTEMPTS == len(bypass.BYPASS_METHODS) + 1
|
|
assert bypass._BYPASS_METHOD_ATTEMPTS >= (
|
|
max(bypass.MAX_CONSECUTIVE_SAME_CHALLENGE, len(bypass.BYPASS_METHODS) + 1)
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# A passive challenge gets an undisturbed window first
|
|
# --------------------------------------------------------------------------- #
|
|
async def _never_passes(*_args, **_kwargs) -> bool:
|
|
return False
|
|
|
|
|
|
def test_passive_challenge_is_given_time_before_any_method_runs(monkeypatch, bypass):
|
|
"""DDoS-Guard's JS check clears itself; nothing should click or reload first."""
|
|
calls: list[str] = []
|
|
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
|
_stub_page_state(monkeypatch, bypass)
|
|
|
|
async def _passes(*_args, **_kwargs) -> bool:
|
|
return True
|
|
|
|
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _passes)
|
|
|
|
assert asyncio.run(bypass._bypass(object())) is True
|
|
assert calls == [], "the page must not be touched while the check can still pass"
|
|
|
|
|
|
def test_passive_wait_happens_once_not_before_every_method(monkeypatch, bypass):
|
|
"""It is a settling window, not a delay bolted onto each attempt."""
|
|
waits: list[int] = []
|
|
calls: list[str] = []
|
|
|
|
async def _count_wait(*_args, **_kwargs) -> bool:
|
|
waits.append(1)
|
|
return False
|
|
|
|
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
|
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _count_wait)
|
|
_stub_page_state(monkeypatch, bypass)
|
|
|
|
asyncio.run(bypass._bypass(object()))
|
|
|
|
assert len(waits) == 1
|
|
assert calls == ["m0", "m1", "m2", "m3"]
|
|
|
|
|
|
def test_no_passive_wait_when_no_challenge_is_detected(monkeypatch, bypass):
|
|
"""The 'none' branch has its own settle-and-refresh handling."""
|
|
waits: list[int] = []
|
|
|
|
async def _count_wait(*_args, **_kwargs) -> bool:
|
|
waits.append(1)
|
|
return False
|
|
|
|
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _count_wait)
|
|
_stub_page_state(monkeypatch, bypass, challenge="none")
|
|
|
|
class _Page:
|
|
async def reload(self, **_kwargs) -> None:
|
|
return None
|
|
|
|
asyncio.run(bypass._bypass(_Page(), max_retries=1))
|
|
|
|
assert waits == []
|
|
|
|
|
|
def test_wait_for_passive_solve_returns_as_soon_as_the_page_clears(monkeypatch, bypass):
|
|
polls = {"n": 0}
|
|
|
|
async def _is_bypassed(*_args, **_kwargs) -> bool:
|
|
polls["n"] += 1
|
|
return polls["n"] >= 3
|
|
|
|
monkeypatch.setattr(bypass, "_is_bypassed", _is_bypassed)
|
|
|
|
assert asyncio.run(bypass._wait_for_passive_solve(object())) is True
|
|
assert polls["n"] == 3
|
|
|
|
|
|
def test_wait_for_passive_solve_gives_up_at_the_window(monkeypatch, bypass):
|
|
"""It must not poll forever - the methods still need their share of the budget."""
|
|
clock = {"now": 0.0}
|
|
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
|
|
|
async def _tick(*_args, **_kwargs) -> bool:
|
|
clock["now"] += 1.0
|
|
return False
|
|
|
|
monkeypatch.setattr(bypass, "_is_bypassed", _tick)
|
|
|
|
assert asyncio.run(bypass._wait_for_passive_solve(object())) is False
|
|
assert clock["now"] >= bypass._PASSIVE_SOLVE_SECONDS
|
|
|
|
|
|
def test_wait_for_passive_solve_honours_cancellation(monkeypatch, bypass):
|
|
import threading
|
|
|
|
from shelfmark.bypass import BypassCancelledError
|
|
|
|
cancel = threading.Event()
|
|
cancel.set()
|
|
monkeypatch.setattr(bypass, "_is_bypassed", _never_passes)
|
|
|
|
with pytest.raises(BypassCancelledError):
|
|
asyncio.run(bypass._wait_for_passive_solve(object(), cancel))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# The page-load loop stops while there is still time to report a real failure
|
|
# --------------------------------------------------------------------------- #
|
|
def test_page_load_loop_stops_before_the_worker_deadline(monkeypatch, bypass):
|
|
"""A stubborn challenge must produce "bypass failed", not a cancelled coroutine."""
|
|
clock = {"now": 0.0}
|
|
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
|
monkeypatch.delenv(bypass._BYPASS_CHILD_ENV, raising=False)
|
|
|
|
attempts = {"n": 0}
|
|
|
|
async def _create(_url):
|
|
return object()
|
|
|
|
async def _get(_url, _driver, _cancel=None) -> str:
|
|
attempts["n"] += 1
|
|
# Each pass eats a realistic slice of the budget.
|
|
clock["now"] += 120.0
|
|
return ""
|
|
|
|
async def _close(_driver) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(bypass, "_create_cdp_browser", _create)
|
|
monkeypatch.setattr(bypass, "_get", _get)
|
|
monkeypatch.setattr(bypass, "_close_cdp_driver", _close)
|
|
|
|
class _RealWorker:
|
|
def run(self, coro, timeout=None):
|
|
return asyncio.run(coro)
|
|
|
|
monkeypatch.setattr(bypass, "_CDP_WORKER", _RealWorker())
|
|
|
|
result = bypass._run_bypass_in_current_process("https://example.com", 10)
|
|
|
|
assert result == ""
|
|
# Well short of the 10 it was asked for, and short of the deadline it had.
|
|
assert attempts["n"] < 10
|
|
budget = bypass._IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
|
assert clock["now"] < budget, "the loop must leave room to report the failure"
|
|
|
|
|
|
def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, bypass):
|
|
"""The deadline check must never skip the request entirely."""
|
|
clock = {"now": 10_000.0}
|
|
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
|
monkeypatch.delenv(bypass._BYPASS_CHILD_ENV, raising=False)
|
|
|
|
attempts = {"n": 0}
|
|
|
|
async def _create(_url):
|
|
return object()
|
|
|
|
async def _get(_url, _driver, _cancel=None) -> str:
|
|
attempts["n"] += 1
|
|
return "<html>solved</html>"
|
|
|
|
async def _close(_driver) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(bypass, "_create_cdp_browser", _create)
|
|
monkeypatch.setattr(bypass, "_get", _get)
|
|
monkeypatch.setattr(bypass, "_close_cdp_driver", _close)
|
|
|
|
class _RealWorker:
|
|
def run(self, coro, timeout=None):
|
|
return asyncio.run(coro)
|
|
|
|
monkeypatch.setattr(bypass, "_CDP_WORKER", _RealWorker())
|
|
|
|
assert bypass._run_bypass_in_current_process("https://example.com", 10) == "<html>solved</html>"
|
|
assert attempts["n"] == 1
|