mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 20:20:20 +01:00
258 lines
8.9 KiB
Python
258 lines
8.9 KiB
Python
"""The wall-clock budget on a release search.
|
|
|
|
Issue #1276: `/api/releases` is synchronous and nothing bounded it, while the bypass path
|
|
it can reach was allowed ~840s per URL. A search that ran into an unsolvable protection
|
|
challenge therefore outlived every reverse proxy in front of it, and the user was shown
|
|
"Server unavailable (504)" - a gateway timeout that names their proxy rather than the
|
|
challenge that actually failed.
|
|
"""
|
|
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from shelfmark.core import search_deadline
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_ambient_deadline():
|
|
"""Each test starts outside any budget."""
|
|
token = search_deadline._current.set(None)
|
|
yield
|
|
search_deadline._current.reset(token)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# The budget itself
|
|
# --------------------------------------------------------------------------- #
|
|
def test_no_budget_outside_a_search():
|
|
"""A queued download must not inherit a search's budget."""
|
|
assert search_deadline.current() is None
|
|
assert search_deadline.expired() is False
|
|
assert search_deadline.cancel_event() is None
|
|
|
|
|
|
def test_budget_applies_inside_the_context_and_not_after():
|
|
with search_deadline.search_deadline(60) as deadline:
|
|
assert search_deadline.current() is deadline
|
|
assert search_deadline.expired() is False
|
|
assert search_deadline.current() is None
|
|
|
|
|
|
def test_expiry_trips_the_cancel_event():
|
|
"""The Event is the mechanism: the bypassers poll it and know nothing of deadlines."""
|
|
with search_deadline.search_deadline(0.05):
|
|
event = search_deadline.cancel_event()
|
|
assert isinstance(event, threading.Event)
|
|
assert event.wait(timeout=5) is True
|
|
assert search_deadline.expired() is True
|
|
|
|
|
|
def test_timer_is_cancelled_on_exit():
|
|
"""A finished search must not leave a timer running to fire later."""
|
|
with search_deadline.search_deadline(3600) as deadline:
|
|
pass
|
|
assert deadline._timer.finished.is_set()
|
|
|
|
|
|
def test_message_names_the_challenge_not_the_proxy():
|
|
with search_deadline.search_deadline(120):
|
|
message = search_deadline.deadline_message()
|
|
assert "120s" in message
|
|
assert "protection challenge" in message
|
|
assert "504" not in message
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Reading the setting
|
|
# --------------------------------------------------------------------------- #
|
|
@pytest.mark.parametrize(
|
|
("configured", "expected"),
|
|
[
|
|
(600, 600.0),
|
|
("450", 450.0),
|
|
(None, search_deadline.DEFAULT_SEARCH_BUDGET_SECONDS),
|
|
("nonsense", search_deadline.DEFAULT_SEARCH_BUDGET_SECONDS),
|
|
(0, search_deadline.DEFAULT_SEARCH_BUDGET_SECONDS),
|
|
(True, search_deadline.DEFAULT_SEARCH_BUDGET_SECONDS),
|
|
(5, 30.0), # clamped up: below this nothing can finish
|
|
(99999, 1800.0), # clamped down
|
|
],
|
|
)
|
|
def test_budget_seconds_coerces_and_clamps(monkeypatch, configured, expected):
|
|
from shelfmark.core.config import config as app_config
|
|
|
|
monkeypatch.setattr(
|
|
app_config,
|
|
"get",
|
|
lambda key, default=None: configured if key == "RELEASE_SEARCH_TIMEOUT" else default,
|
|
)
|
|
|
|
assert search_deadline.budget_seconds() == expected
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# What the search path does with it
|
|
# --------------------------------------------------------------------------- #
|
|
def test_html_get_page_will_not_start_a_bypass_on_a_spent_budget(monkeypatch):
|
|
"""A minutes-long solve nobody is still waiting for is worse than a clear failure."""
|
|
import shelfmark.download.http as http
|
|
|
|
started: list[str] = []
|
|
monkeypatch.setattr(
|
|
http, "get_bypassed_page", lambda *a, **k: started.append(a[0]) or "<html/>"
|
|
)
|
|
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
|
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 1.0)
|
|
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
|
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
|
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
|
|
|
class _Selector:
|
|
current_base = "https://annas-archive.gl"
|
|
attempts_this_dns = 0
|
|
last_failure = None
|
|
|
|
def rewrite(self, url):
|
|
return url
|
|
|
|
with search_deadline.search_deadline(60) as deadline:
|
|
deadline.event.set()
|
|
result = http.html_get_page(
|
|
"https://annas-archive.gl/search",
|
|
retry=1,
|
|
selector=_Selector(),
|
|
use_bypasser=True,
|
|
success_delay=0,
|
|
)
|
|
|
|
assert result == ""
|
|
assert started == [], "no solve should have been started"
|
|
|
|
|
|
def test_search_budget_becomes_the_cancel_flag(monkeypatch):
|
|
"""This is what makes the budget bite on a solve already running."""
|
|
import shelfmark.download.http as http
|
|
|
|
seen: list[object] = []
|
|
|
|
def fake_bypass(_url, _selector=None, cancel_flag=None):
|
|
seen.append(cancel_flag)
|
|
return "<html>solved</html>"
|
|
|
|
monkeypatch.setattr(http, "get_bypassed_page", fake_bypass)
|
|
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
|
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 1.0)
|
|
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
|
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
|
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
|
|
|
class _Selector:
|
|
current_base = "https://annas-archive.gl"
|
|
attempts_this_dns = 0
|
|
last_failure = None
|
|
|
|
def rewrite(self, url):
|
|
return url
|
|
|
|
with search_deadline.search_deadline(60) as deadline:
|
|
http.html_get_page(
|
|
"https://annas-archive.gl/search",
|
|
retry=1,
|
|
selector=_Selector(),
|
|
use_bypasser=True,
|
|
success_delay=0,
|
|
)
|
|
|
|
assert seen == [deadline.event]
|
|
|
|
|
|
def test_a_callers_own_cancel_flag_is_not_replaced(monkeypatch):
|
|
"""A queued download brings its own and must keep it."""
|
|
import shelfmark.download.http as http
|
|
|
|
seen: list[object] = []
|
|
|
|
def fake_bypass(_url, _selector=None, cancel_flag=None):
|
|
seen.append(cancel_flag)
|
|
return "<html>solved</html>"
|
|
|
|
monkeypatch.setattr(http, "get_bypassed_page", fake_bypass)
|
|
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
|
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 1.0)
|
|
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
|
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
|
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
|
|
|
class _Selector:
|
|
current_base = "https://annas-archive.gl"
|
|
attempts_this_dns = 0
|
|
last_failure = None
|
|
|
|
def rewrite(self, url):
|
|
return url
|
|
|
|
own_flag = threading.Event()
|
|
with search_deadline.search_deadline(60):
|
|
http.html_get_page(
|
|
"https://annas-archive.gl/search",
|
|
retry=1,
|
|
selector=_Selector(),
|
|
cancel_flag=own_flag,
|
|
use_bypasser=True,
|
|
success_delay=0,
|
|
)
|
|
|
|
assert seen == [own_flag]
|
|
|
|
|
|
def test_expired_budget_reports_the_challenge_not_a_cancellation(monkeypatch):
|
|
"""The budget trips the same flag a user's cancel does; the messages must differ."""
|
|
import shelfmark.download.http as http
|
|
from shelfmark.bypass import BypassCancelledError
|
|
|
|
def fake_bypass(*_a, **_k):
|
|
msg = "Bypass cancelled"
|
|
raise BypassCancelledError(msg)
|
|
|
|
monkeypatch.setattr(http, "get_bypassed_page", fake_bypass)
|
|
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
|
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 1.0)
|
|
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
|
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
|
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
|
|
|
failures: list[str] = []
|
|
|
|
class _Selector:
|
|
current_base = "https://annas-archive.gl"
|
|
attempts_this_dns = 0
|
|
|
|
def rewrite(self, url):
|
|
return url
|
|
|
|
@property
|
|
def last_failure(self):
|
|
return None
|
|
|
|
@last_failure.setter
|
|
def last_failure(self, value):
|
|
if value:
|
|
failures.append(value)
|
|
|
|
with search_deadline.search_deadline(60) as deadline:
|
|
# Expire mid-solve: the flag is set, but the caller reaches the handler by way of
|
|
# BypassCancelledError, which on its own reads as "someone cancelled this".
|
|
deadline.expires_at = 0.0
|
|
http.html_get_page(
|
|
"https://annas-archive.gl/search",
|
|
retry=1,
|
|
selector=_Selector(),
|
|
use_bypasser=True,
|
|
success_delay=0,
|
|
)
|
|
|
|
assert failures, "a give-up reason should have been recorded"
|
|
assert "ran out of time" in failures[-1]
|
|
assert "cancelled" not in failures[-1]
|