mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-17 08:38:14 +00:00
fix(bypass): read the solved page once it has finished loading (#1721)
After a DDoS-Guard check clears itself, an Anna's Archive search can come back as "No results found" for a book AA has. On 1.7.0: ``` Bypass successful: ddos_guard challenge cleared itself Search page has no results table: https://annas-archive.gl/search?... (bytes=122986, title='... - Search - Anna's Archive', aa_markers=['/md5/', '/dyn/'], challenge_marker=None, over_challenge_size_cap=True) No results table found for query: ... ``` The same URL in a browser is about 745 KB, and the first `<table>` sits near byte 270,000. `_read_page_source` returns the `html` element as soon as it exists, which is as soon as the first bytes are parsed. AA streams the head and filters before the results, and the passive-solve check is satisfied by 50 characters of body text, so the read can land mid-document. Here it was one bypassed search in eight. This waits, inside the same `BYPASS_PAGE_SOURCE_TIMEOUT` budget, for `document.readyState` to leave `loading`, and reads state and HTML in one `evaluate` (same reasoning as the atomic snapshot in `_wait_for_aa_download_page`). If the budget runs out or the reads keep failing, it returns the element's HTML as before and logs a warning. Tests are in `tests/bypass/test_bypass_budgets.py`, next to the existing page-source ones. With the source change reverted the three new tests fail (two get the first chunk instead of the whole page). `make python-checks` is clean and `tests/bypass` passes. I also reproduced it with a real Chromium against a local server that serves a challenge page, then the page in two chunks 6 s apart: unpatched, 7,244 of 7,311 bytes and no table; patched, the whole page. I can share the script. One consequence of the old behaviour: the half-read page raises instead of returning an empty result, so the retry without the language filter never runs. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> <!-- coderabbit-ai-generated:v1 --> ## Summary by CodeRabbit * **Bug Fixes** * Page content is captured after the document finishes loading, improving the completeness of results. * If loading takes too long or a read fails, the current page content is used as a fallback so processing can continue. * If the page changes while content is being read, the fallback uses a fresh page reference to retrieve the available content. * Page-source reads continue to respect the configured timeout. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
1 parent
7f4d20ed9d
commit
9b89857923
2 files changed
+201
-4
No files matched your search
@@ -91,6 +91,22 @@ _HELPER_IDLE_TIMEOUT_DEFAULT = 180.0
|
||||
# How long to wait for a solved page to produce its document before the attempt is
|
||||
# abandoned. SeleniumBase's own get_page_source() allows one second; see _read_page_source.
|
||||
_PAGE_SOURCE_TIMEOUT_DEFAULT = 20.0
|
||||
# How often to ask whether a solved page has finished parsing, inside that same budget.
|
||||
_PAGE_LOAD_POLL_SECONDS = 0.25
|
||||
# Consecutive failed reads of the document state before falling back to the element read.
|
||||
# A page whose CDP calls keep failing will not start answering, and every retry spends
|
||||
# budget the caller needs.
|
||||
_PAGE_LOAD_MAX_READ_ERRORS = 3
|
||||
# The least time the fallback gets to look the `html` element up again, even when the
|
||||
# budget is spent: enough for one real lookup on a page that is there. It matches
|
||||
# SeleniumBase's own default for find().
|
||||
_PAGE_SOURCE_FALLBACK_FIND_SECONDS = 1.0
|
||||
# One CDP call, not two: a navigation can commit between a readiness read and an HTML
|
||||
# read, so the old document would pass the check and the first chunk of the new one would
|
||||
# be returned. Same reasoning as the atomic snapshot in _wait_for_aa_download_page.
|
||||
_LOADED_DOCUMENT_EXPRESSION = (
|
||||
"document.readyState === 'loading' ? '' : document.documentElement.outerHTML"
|
||||
)
|
||||
# Leave time inside the existing browser watchdog for challenge solving and cleanup.
|
||||
_AA_WAITING_ROOM_TIMEOUT_SECONDS = 300.0
|
||||
_AA_WAITING_ROOM_POLL_SECONDS = 1.0
|
||||
@@ -855,15 +871,62 @@ async def _read_page_source(page: Any) -> str:
|
||||
challenge is often still navigating to the real content, so the read times out even
|
||||
though the solve succeeded: the whole attempt is retried, and the repeated requests
|
||||
are what earn a 429 from a host that was about to serve us.
|
||||
|
||||
The `html` element existing is not enough either: it is there as soon as the first
|
||||
bytes are parsed. Anna's Archive streams the head and filters of a search page before
|
||||
the results, and the bypass check passes on that prefix, so reading the element at
|
||||
once returned about 120 KB of a 745 KB page with no results table in it. So within the
|
||||
same budget, wait for `document.readyState` to leave "loading" and read the whole
|
||||
document. If the budget runs out or the reads keep failing, look the `html` element
|
||||
up again and return what it holds, which is what this returned before.
|
||||
"""
|
||||
timeout = _coerce_non_negative_float(
|
||||
app_config.get("BYPASS_PAGE_SOURCE_TIMEOUT", _PAGE_SOURCE_TIMEOUT_DEFAULT),
|
||||
_PAGE_SOURCE_TIMEOUT_DEFAULT,
|
||||
)
|
||||
deadline = time.monotonic() + timeout
|
||||
element = await page.find("html", timeout=timeout)
|
||||
html = await _read_loaded_document(page, deadline)
|
||||
if html is not None:
|
||||
return html
|
||||
# The page may have navigated while we polled, and the element found above would then
|
||||
# belong to the document it left: reading it raises a stale-node error. Look it up
|
||||
# again in whatever document is there now.
|
||||
remaining = max(deadline - time.monotonic(), _PAGE_SOURCE_FALLBACK_FIND_SECONDS)
|
||||
element = await page.find("html", timeout=remaining)
|
||||
return await element.get_html_async()
|
||||
|
||||
|
||||
async def _read_loaded_document(page: Any, deadline: float) -> str | None:
|
||||
"""Return the document's HTML once it has finished parsing, or None by `deadline`.
|
||||
|
||||
The document is read at least once, so a spent budget still returns a page that has
|
||||
already loaded.
|
||||
"""
|
||||
errors = 0
|
||||
while True:
|
||||
try:
|
||||
html = await page.evaluate(_LOADED_DOCUMENT_EXPRESSION)
|
||||
# An empty string means still loading. SeleniumBase's evaluate() returns None
|
||||
# when CDP sends back no value. Only a non-empty string is a document.
|
||||
if isinstance(html, str) and html:
|
||||
return html
|
||||
errors = 0
|
||||
except _CDP_OPERATION_ERRORS as e:
|
||||
errors += 1
|
||||
logger.debug("Reading the loaded document failed (%d): %s", errors, e)
|
||||
if errors >= _PAGE_LOAD_MAX_READ_ERRORS:
|
||||
logger.warning(
|
||||
"Could not read the loaded document after %d attempts; reading the page as it is",
|
||||
errors,
|
||||
)
|
||||
return None
|
||||
if time.monotonic() >= deadline:
|
||||
logger.warning("Page still loading after the page-source budget; reading it as it is")
|
||||
return None
|
||||
await asyncio.sleep(_PAGE_LOAD_POLL_SECONDS)
|
||||
|
||||
|
||||
async def _read_waiting_room_snapshot(
|
||||
page: Any, cancel_flag: Event | None
|
||||
) -> _WaitingRoomSnapshot | None:
|
||||
|
||||
@@ -268,24 +268,62 @@ def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, b
|
||||
assert attempts["n"] == 1
|
||||
|
||||
|
||||
_LOADED_DOCUMENT_EXPRESSION = (
|
||||
"document.readyState === 'loading' ? '' : document.documentElement.outerHTML"
|
||||
)
|
||||
|
||||
|
||||
class _FakeElement:
|
||||
def __init__(self, html: str = "<html>solved</html>") -> None:
|
||||
self.html = html
|
||||
|
||||
async def get_html_async(self) -> str:
|
||||
return "<html>solved</html>"
|
||||
return self.html
|
||||
|
||||
|
||||
class _FakePage:
|
||||
"""A page that only produces its document after `ready_after` seconds of waiting."""
|
||||
"""A page that only produces its document after `ready_after` seconds of waiting.
|
||||
|
||||
def __init__(self, ready_after: float = 0.0) -> None:
|
||||
`states` scripts the document's readyState, one per read, the last one repeating. An
|
||||
entry that is an exception is raised instead. A read while "loading" answers "", as
|
||||
the real expression does; any other state answers `full`. The `html` element always
|
||||
holds `partial`, so a test that gets `full` cannot have passed through the fallback.
|
||||
The fixture removes sleeps, so the page progresses per read rather than per second.
|
||||
Any other expression raises AssertionError, which the code under test does not catch,
|
||||
so a typo in the expression fails the test instead of being absorbed.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ready_after: float = 0.0,
|
||||
*,
|
||||
states: list[object] | None = None,
|
||||
partial: str = "<html>solved</html>",
|
||||
full: str = "<html>solved</html>",
|
||||
) -> None:
|
||||
self.ready_after = ready_after
|
||||
self.waited_with: list[float] = []
|
||||
self.states = states if states is not None else ["complete"]
|
||||
self.partial = partial
|
||||
self.full = full
|
||||
self.state_polls = 0
|
||||
|
||||
async def find(self, selector: str, timeout: float = 1):
|
||||
self.waited_with.append(timeout)
|
||||
if timeout < self.ready_after:
|
||||
msg = f"Time ran out while waiting for: {{{selector}}}"
|
||||
raise TimeoutError(msg)
|
||||
return _FakeElement()
|
||||
return _FakeElement(self.partial)
|
||||
|
||||
async def evaluate(self, expression: str) -> object:
|
||||
if expression != _LOADED_DOCUMENT_EXPRESSION:
|
||||
msg = f"unexpected expression: {expression}"
|
||||
raise AssertionError(msg)
|
||||
state = self.states[min(self.state_polls, len(self.states) - 1)]
|
||||
self.state_polls += 1
|
||||
if isinstance(state, Exception):
|
||||
raise state
|
||||
return "" if state == "loading" else self.full
|
||||
|
||||
|
||||
def test_page_source_waits_longer_than_seleniumbases_one_second(bypass):
|
||||
@@ -313,3 +351,99 @@ def test_page_source_timeout_is_configurable(bypass, monkeypatch):
|
||||
asyncio.run(bypass._read_page_source(page))
|
||||
|
||||
assert page.waited_with == [45.0]
|
||||
|
||||
|
||||
_PARTIAL_SEARCH_PAGE = "<html><head><title>Search</title></head><body><div>filters"
|
||||
_FULL_SEARCH_PAGE = _PARTIAL_SEARCH_PAGE + "<table>results</table></div></body></html>"
|
||||
|
||||
|
||||
def test_page_source_waits_for_the_document_to_finish_loading(bypass):
|
||||
"""The `html` element exists before the page has arrived.
|
||||
|
||||
On 1.7.0 a cleared DDoS-Guard check returned 122,986 bytes of an Anna's Archive search
|
||||
page of about 745,000, read while the results were still streaming in, and the user
|
||||
was shown "No results found".
|
||||
"""
|
||||
page = _FakePage(
|
||||
states=["loading", "loading", "loading", "interactive"],
|
||||
partial=_PARTIAL_SEARCH_PAGE,
|
||||
full=_FULL_SEARCH_PAGE,
|
||||
)
|
||||
|
||||
assert asyncio.run(bypass._read_page_source(page)) == _FULL_SEARCH_PAGE
|
||||
assert page.state_polls == 4
|
||||
|
||||
|
||||
def test_page_source_falls_back_to_the_element_when_loading_outlasts_the_budget(
|
||||
monkeypatch, bypass
|
||||
):
|
||||
"""A page that never finishes loading must be read as before, not raise or hang."""
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
page = _FakePage(states=["loading"], partial=_PARTIAL_SEARCH_PAGE, full=_FULL_SEARCH_PAGE)
|
||||
original_evaluate = page.evaluate
|
||||
|
||||
async def _evaluate_slowly(expression: str) -> object:
|
||||
clock["now"] += 1.0
|
||||
return await original_evaluate(expression)
|
||||
|
||||
page.evaluate = _evaluate_slowly
|
||||
|
||||
assert asyncio.run(bypass._read_page_source(page)) == _PARTIAL_SEARCH_PAGE
|
||||
assert clock["now"] >= bypass._PAGE_SOURCE_TIMEOUT_DEFAULT
|
||||
assert page.state_polls <= bypass._PAGE_SOURCE_TIMEOUT_DEFAULT + 1
|
||||
# The budget is spent, so the second lookup gets the floor, not a negative timeout.
|
||||
assert page.waited_with == [
|
||||
bypass._PAGE_SOURCE_TIMEOUT_DEFAULT,
|
||||
bypass._PAGE_SOURCE_FALLBACK_FIND_SECONDS,
|
||||
]
|
||||
|
||||
|
||||
class _StaleNodeElement:
|
||||
"""An element of a document the page has left: CDP no longer knows its node."""
|
||||
|
||||
async def get_html_async(self) -> str:
|
||||
from seleniumbase.undetected.cdp_driver.connection import ProtocolException
|
||||
|
||||
raise ProtocolException("No node with given id found")
|
||||
|
||||
|
||||
def test_page_source_fallback_reads_the_document_the_page_is_on_now(monkeypatch, bypass):
|
||||
"""A navigation during the wait leaves the first `html` element stale.
|
||||
|
||||
Reading it raised a CDP error out of _read_page_source and failed the attempt, when
|
||||
the page had a document to return.
|
||||
"""
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
page = _FakePage(states=["loading"], partial=_PARTIAL_SEARCH_PAGE, full=_FULL_SEARCH_PAGE)
|
||||
original_evaluate = page.evaluate
|
||||
elements: list[object] = [_StaleNodeElement(), _FakeElement("<html>new page</html>")]
|
||||
|
||||
async def _evaluate_slowly(expression: str) -> object:
|
||||
clock["now"] += 1.0
|
||||
return await original_evaluate(expression)
|
||||
|
||||
async def _find(selector: str, timeout: float = 1) -> object:
|
||||
page.waited_with.append(timeout)
|
||||
return elements.pop(0)
|
||||
|
||||
page.evaluate = _evaluate_slowly
|
||||
page.find = _find
|
||||
|
||||
assert asyncio.run(bypass._read_page_source(page)) == "<html>new page</html>"
|
||||
assert elements == []
|
||||
|
||||
|
||||
def test_page_source_survives_a_cdp_error_while_waiting(bypass):
|
||||
"""One failed state read must not drop the page back to its half-loaded prefix."""
|
||||
from seleniumbase.undetected.cdp_driver.connection import ProtocolException
|
||||
|
||||
page = _FakePage(
|
||||
states=[ProtocolException("Execution context was destroyed"), "loading", "complete"],
|
||||
partial=_PARTIAL_SEARCH_PAGE,
|
||||
full=_FULL_SEARCH_PAGE,
|
||||
)
|
||||
|
||||
assert asyncio.run(bypass._read_page_source(page)) == _FULL_SEARCH_PAGE
|
||||
assert page.state_polls == 3
|
||||
Reference in new issue
Block a user