Files
shelfmark/tests/download/test_search_warmup.py
T
CaliBrain ebb833a82c fix(bypass): discard rejected DDoS-Guard cookies instead of replaying them (#1221)
A cookie that has been rejected was kept and presented again on every
later
request, so a single bad clearance could re-arm the challenge
indefinitely.

Cookie storage:
- Enforce expiry for every stored cookie, not just cf_clearance.
DDoS-Guard
domains have no cf_clearance, so the existing check never fired for them
and
  expired cookies were replayed forever.
- Stop storing the per-check cookies __ddg8_/__ddg9_/__ddg10_ and
ddg_last_challenge. Captured live from Anna's Archive, these carry the
client
IP and the timestamp the check was issued (~40 min), versus ~1 year for
the
  __ddg1_/__ddg2_/__ddgid_ clearance. Replaying an IP-bound token stops
describing the caller as soon as the egress IP changes, which is routine
  behind a VPN.

Failure handling — every path that is rejected while carrying cookies
now
purges them, not just the redirect loop:
- 403 returned while presenting cookies.
- Cached-cookie attempt rejected, whether by status or by redirect loop.
- Factored the purge into _purge_clearance, guarded on a non-empty
hostname
since clear_cf_cookies("") means "every host" and would wipe clearance
for
  sites that are working fine.

Also fix the search warm-up switches shipped inert in v1.3.8:
SEARCH_WARMUP_ENABLED and SEARCH_WARMUP_QUERY are not in the settings
registry, and config.get only consults the environment for keys it
knows, so
both always returned their defaults — the warm-up could not be turned
off or
retargeted. Read os.environ first.

Refs #1220. Deliberately not "Fixes": the reported failure could not be
reproduced on v1.3.8 from a stable IP (the reporter's own queries all
returned
200 on both the pre- and post-change builds), and the new purge paths
did not
fire in live testing because the failures arrive as redirect loops,
which were
already purged. These are correctness fixes with no measured effect on
that
issue. The underlying problem remains that Chrome-obtained cookies never
satisfy DDoS-Guard when replayed by requests, so every search still
re-solves.

Verified: 2542 unit tests pass; ruff, basedpyright and vulture clean;
e2e
platform baseline (10), full (6) and bypasser-external (5) all pass;
five
sequential live searches against Anna's Archive all returned 200 with
zero
"Exceeded 30 redirects".
2026-08-15 17:08:11 -04:00

171 lines
5.5 KiB
Python

"""Boot-time search warm-up.
The warm-up exists to move the cold DDoS-Guard solve off the user's first search. It
is an optimisation, so the load-bearing property is that it can never affect startup:
a source that is down, misconfigured or raising must leave the app running.
"""
import pytest
@pytest.fixture
def warmup(monkeypatch):
import shelfmark.download.warmup as warmup_module
monkeypatch.setattr(warmup_module, "_warmup_thread", None)
return warmup_module
def _patch_config(monkeypatch, warmup, values: dict):
def fake_get(key, default=None):
return values.get(key, default)
monkeypatch.setattr(warmup.config, "get", fake_get)
def test_disabled_by_setting(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": False})
assert warmup.is_enabled() is False
assert warmup.start() is False
def test_disabled_by_string_false(monkeypatch, warmup):
"""Deployment ENV arrives as a string, not a bool."""
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": "false"})
assert warmup.is_enabled() is False
def test_skipped_when_direct_download_is_off(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {"DIRECT_DOWNLOAD_ENABLED": False})
assert warmup.is_enabled() is False
def test_enabled_by_default(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
assert warmup.is_enabled() is True
def test_query_defaults_and_is_configurable(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
assert warmup.warmup_query() == "The Great Gatsby"
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "Dune"})
assert warmup.warmup_query() == "Dune"
# A blank override must not send an empty query at the source.
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": " "})
assert warmup.warmup_query() == "The Great Gatsby"
def test_skipped_when_no_mirrors_configured(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
import shelfmark.core.mirrors as mirrors
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: False)
called: list[str] = []
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(dd, "search_books", lambda q, f: called.append(q))
assert warmup.run_warmup() is False
assert called == []
def test_successful_warmup_reports_true(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
import shelfmark.core.mirrors as mirrors
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
seen: list[str] = []
def fake_search(query, _filters):
seen.append(query)
return ["a", "b"]
monkeypatch.setattr(dd, "search_books", fake_search)
assert warmup.run_warmup() is True
assert seen == ["The Great Gatsby"]
def test_empty_results_are_not_an_error(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
import shelfmark.core.mirrors as mirrors
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
monkeypatch.setattr(dd, "search_books", lambda q, f: [])
assert warmup.run_warmup() is False
def test_search_failure_is_swallowed(monkeypatch, warmup):
"""A source that is down at boot must not propagate out of the warm-up."""
_patch_config(monkeypatch, warmup, {})
import shelfmark.core.mirrors as mirrors
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
def boom(_query, _filters):
msg = "mirrors are blocked"
raise RuntimeError(msg)
monkeypatch.setattr(dd, "search_books", boom)
assert warmup.run_warmup() is False
def test_start_schedules_a_daemon_thread_and_is_idempotent(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
assert warmup.start(delay_seconds=30) is True
thread = warmup._warmup_thread
assert thread is not None
assert thread.daemon is True
# A second call must not stack up another timer.
assert warmup.start(delay_seconds=30) is False
assert warmup._warmup_thread is thread
thread.cancel()
def test_start_does_not_run_the_search_inline(monkeypatch, warmup):
"""Startup must not block on a search that can take a minute."""
_patch_config(monkeypatch, warmup, {})
ran: list[bool] = []
monkeypatch.setattr(warmup, "run_warmup", lambda: ran.append(True))
warmup.start(delay_seconds=30)
assert ran == []
if warmup._warmup_thread:
warmup._warmup_thread.cancel()
def test_env_var_can_disable_the_warmup(monkeypatch, warmup):
"""SEARCH_WARMUP_ENABLED is not in the settings registry, so config.get never
sees it - the documented off-switch only works if os.environ is consulted."""
_patch_config(monkeypatch, warmup, {}) # config knows nothing about the key
monkeypatch.setenv("SEARCH_WARMUP_ENABLED", "false")
assert warmup.is_enabled() is False
assert warmup.start() is False
def test_env_var_can_set_the_query(monkeypatch, warmup):
_patch_config(monkeypatch, warmup, {})
monkeypatch.setenv("SEARCH_WARMUP_QUERY", "Moby Dick")
assert warmup.warmup_query() == "Moby Dick"
def test_env_var_absent_falls_back_to_config(monkeypatch, warmup):
monkeypatch.delenv("SEARCH_WARMUP_QUERY", raising=False)
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "From Config"})
assert warmup.warmup_query() == "From Config"