From d7fe28595c6e9c72229b21a0b05033b5549ac68a Mon Sep 17 00:00:00 2001 From: Jorge Lima <5619521+jfmlima@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:17:15 +0100 Subject: [PATCH] fix(bypass): wait for the solved page before reading its source (#1286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #1276 with a measurement from the instance I reported there. v1.3.13 solves the challenge again, but on my setup the solve was being thrown away immediately afterwards: ``` 19:26:08 Bypass successful using _bypass_method_cdp_gui_click 19:26:16 Bypass failed (attempt 1/10): TimeoutError: Time ran out while waiting for: {html} ``` `_get()` ends with `return await page.get_page_source()`, which is `find("html", timeout=1)` in SeleniumBase. One second is enough for a page that is already sitting on its content, but Anna's Archive answers a cleared check with a redirect to the real page, so the document is not there yet. The solve is discarded, the whole attempt restarts, and the extra requests are what earn the 429 that `note_rate_limited()` then parks the host for — 120 s, then 300 s. ## Change `_read_page_source()` waits for the document itself, with a `BYPASS_PAGE_SOURCE_TIMEOUT` setting (default 20 s, min 1, max 120) in Direct Download → Cloudflare Bypass, next to the existing bypasser timeouts. ## Measured on a live instance I patched the wait in the running container (`find("html", timeout=1)` → `timeout=20` in the installed seleniumbase, which is the same effect as this PR) and re-ran the same searches on the same host, k3s behind a Surfshark WireGuard exit, internal bypasser, v1.3.13: | | 1 s wait | 20 s wait | |---|---|---| | `Time ran out while waiting for: {html}` | one per solve | none | | 429 backoffs | 2 (120 s, then 300 s) | none | | Search for a book AA has | 199 s and 200 s, both errored | 61 s, 2 epub releases | A download after that took 5 s from LibGen, so the search was the whole cost. ## Tests Two tests in `tests/bypass/test_bypass_budgets.py`, the file already covering #1276: a page that needs longer than a second still yields its HTML, and `BYPASS_PAGE_SOURCE_TIMEOUT` overrides the default. `uv run pytest tests/ --ignore=tests/e2e`: 2848 passed, 47 skipped. Ruff check and format clean. The docs table is auto-generated, but running `scripts/generate_env_docs.py` here rewrote unrelated entries (Newznab, BOOK_LANGUAGE), so I added only the new entry by hand in the generator's format rather than commit that churn. One thing I could not judge from outside: whether 20 s is the right default for hosts other than AA. It only costs anything when a solve would otherwise be discarded, but I have measured it on one site. --- docs/environment-variables.md | 11 +++++++ shelfmark/bypass/internal_bypasser.py | 21 +++++++++++- shelfmark/config/settings.py | 12 +++++++ tests/bypass/test_bypass_budgets.py | 47 +++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 4904d7e..c78cd22 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -2344,6 +2344,7 @@ Override destination based on content type metadata. | `EXT_BYPASSER_URL` | URL of the external bypasser service (e.g., FlareSolverr). | string | `http://flaresolverr:8191` | | `EXT_BYPASSER_PATH` | API path for the external bypasser. | string | `/v1` | | `EXT_BYPASSER_TIMEOUT` | Timeout for external bypasser requests in milliseconds. | number | `60000` | +| `BYPASS_PAGE_SOURCE_TIMEOUT` | How long to wait for a solved page to produce its content before the bypass is retried. Raise it if solves succeed but searches still fail. | number | `20` | | `BYPASS_BROWSER_IDLE_TIMEOUT` | How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner. | number | `180` |
@@ -2400,6 +2401,16 @@ Timeout for external bypasser requests in milliseconds. - **Requires restart:** Yes - **Constraints:** min: 10000, max: 300000 +#### `BYPASS_PAGE_SOURCE_TIMEOUT` + +**Page Read Timeout (seconds)** + +How long to wait for a solved page to produce its content before the bypass is retried. Raise it if solves succeed but searches still fail. + +- **Type:** number +- **Default:** `20` +- **Constraints:** min: 1, max: 120 + #### `BYPASS_BROWSER_IDLE_TIMEOUT` **Bypasser Idle Timeout (seconds)** diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index 1536e50..0daba9d 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -82,6 +82,9 @@ _HELPER_RESULT_POLL_SECONDS = 0.05 # what it is doing and exit before its session is killed instead. _HELPER_SHUTDOWN_GRACE_SECONDS = 15.0 _HELPER_IDLE_TIMEOUT_DEFAULT = 180.0 +# How long to wait for a solved page to produce its document before the attempt is +# abandoned. SeleniumBase's own get_page_source() allows one second; see _read_page_source. +_PAGE_SOURCE_TIMEOUT_DEFAULT = 20.0 _PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0 # How much of ffmpeg's stderr to quote when reporting that it died. _FFMPEG_ERROR_TAIL_CHARS = 500 @@ -819,6 +822,22 @@ def _build_host_resolver_rules() -> list[str]: DRIVER_RESET_ERRORS = {"ProtocolException", "RuntimeError", "TimeoutError"} +async def _read_page_source(page: Any) -> str: + """Read a solved page's HTML, waiting for the document to arrive. + + `get_page_source()` waits one second for the `html` element. A page released from a + challenge is often still navigating to the real content, so the read times out even + though the solve succeeded: the whole attempt is retried, and the repeated requests + are what earn a 429 from a host that was about to serve us. + """ + timeout = _coerce_non_negative_float( + app_config.get("BYPASS_PAGE_SOURCE_TIMEOUT", _PAGE_SOURCE_TIMEOUT_DEFAULT), + _PAGE_SOURCE_TIMEOUT_DEFAULT, + ) + element = await page.find("html", timeout=timeout) + return await element.get_html_async() + + async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str: """Fetch URL with Cloudflare bypass using a CDP browser.""" _check_cancellation(cancel_flag, "Bypass cancelled before starting") @@ -842,7 +861,7 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str: logger.debug("Starting bypass process...") if await _bypass(page, cancel_flag=cancel_flag): await _extract_cookies_from_cdp(driver, page, url) - return await page.get_page_source() + return await _read_page_source(page) logger.warning("Bypass completed but page still shows protection") try: diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index bfe85f2..07113fd 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -1683,6 +1683,18 @@ def cloudflare_bypass_settings() -> list[SettingsField]: requires_restart=True, show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True}, ), + NumberField( + key="BYPASS_PAGE_SOURCE_TIMEOUT", + label="Page Read Timeout (seconds)", + description=( + "How long to wait for a solved page to produce its content before the " + "bypass is retried. Raise it if solves succeed but searches still fail." + ), + default=20, + min_value=1, + max_value=120, + show_when={"field": "USING_EXTERNAL_BYPASSER", "value": False}, + ), NumberField( key="BYPASS_BROWSER_IDLE_TIMEOUT", label="Bypasser Idle Timeout (seconds)", diff --git a/tests/bypass/test_bypass_budgets.py b/tests/bypass/test_bypass_budgets.py index a526ead..b7cabb9 100644 --- a/tests/bypass/test_bypass_budgets.py +++ b/tests/bypass/test_bypass_budgets.py @@ -266,3 +266,50 @@ def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, b assert bypass._run_bypass_in_current_process("https://example.com", 10) == "solved" assert attempts["n"] == 1 + + +class _FakeElement: + async def get_html_async(self) -> str: + return "solved" + + +class _FakePage: + """A page that only produces its document after `ready_after` seconds of waiting.""" + + def __init__(self, ready_after: float = 0.0) -> None: + self.ready_after = ready_after + self.waited_with: list[float] = [] + + async def find(self, selector: str, timeout: float = 1): + self.waited_with.append(timeout) + if timeout < self.ready_after: + msg = f"Time ran out while waiting for: {{{selector}}}" + raise TimeoutError(msg) + return _FakeElement() + + +def test_page_source_waits_longer_than_seleniumbases_one_second(bypass): + """A page still navigating after a solve must not lose the solve. + + SeleniumBase's get_page_source() allows one second for the document. Anna's Archive + hands back a redirect to the real content instead, so the read raised TimeoutError + while the challenge had in fact been cleared. + """ + page = _FakePage(ready_after=5.0) + + assert asyncio.run(bypass._read_page_source(page)) == "solved" + assert page.waited_with == [bypass._PAGE_SOURCE_TIMEOUT_DEFAULT] + + +def test_page_source_timeout_is_configurable(bypass, monkeypatch): + """BYPASS_PAGE_SOURCE_TIMEOUT overrides the default for slow or fast setups.""" + monkeypatch.setattr( + bypass.app_config, + "get", + lambda key, default=None: 45 if key == "BYPASS_PAGE_SOURCE_TIMEOUT" else default, + ) + page = _FakePage() + + asyncio.run(bypass._read_page_source(page)) + + assert page.waited_with == [45.0]