diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index c4214a8f..dcbb994d 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -91,6 +91,22 @@ _HELPER_IDLE_TIMEOUT_DEFAULT = 180.0 # How long to wait for a solved page to produce its document before the attempt is # abandoned. SeleniumBase's own get_page_source() allows one second; see _read_page_source. _PAGE_SOURCE_TIMEOUT_DEFAULT = 20.0 +# How often to ask whether a solved page has finished parsing, inside that same budget. +_PAGE_LOAD_POLL_SECONDS = 0.25 +# Consecutive failed reads of the document state before falling back to the element read. +# A page whose CDP calls keep failing will not start answering, and every retry spends +# budget the caller needs. +_PAGE_LOAD_MAX_READ_ERRORS = 3 +# The least time the fallback gets to look the `html` element up again, even when the +# budget is spent: enough for one real lookup on a page that is there. It matches +# SeleniumBase's own default for find(). +_PAGE_SOURCE_FALLBACK_FIND_SECONDS = 1.0 +# One CDP call, not two: a navigation can commit between a readiness read and an HTML +# read, so the old document would pass the check and the first chunk of the new one would +# be returned. Same reasoning as the atomic snapshot in _wait_for_aa_download_page. +_LOADED_DOCUMENT_EXPRESSION = ( + "document.readyState === 'loading' ? '' : document.documentElement.outerHTML" +) # Leave time inside the existing browser watchdog for challenge solving and cleanup. _AA_WAITING_ROOM_TIMEOUT_SECONDS = 300.0 _AA_WAITING_ROOM_POLL_SECONDS = 1.0 @@ -855,15 +871,62 @@ async def _read_page_source(page: Any) -> str: challenge is often still navigating to the real content, so the read times out even though the solve succeeded: the whole attempt is retried, and the repeated requests are what earn a 429 from a host that was about to serve us. + + The `html` element existing is not enough either: it is there as soon as the first + bytes are parsed. Anna's Archive streams the head and filters of a search page before + the results, and the bypass check passes on that prefix, so reading the element at + once returned about 120 KB of a 745 KB page with no results table in it. So within the + same budget, wait for `document.readyState` to leave "loading" and read the whole + document. If the budget runs out or the reads keep failing, look the `html` element + up again and return what it holds, which is what this returned before. """ timeout = _coerce_non_negative_float( app_config.get("BYPASS_PAGE_SOURCE_TIMEOUT", _PAGE_SOURCE_TIMEOUT_DEFAULT), _PAGE_SOURCE_TIMEOUT_DEFAULT, ) + deadline = time.monotonic() + timeout element = await page.find("html", timeout=timeout) + html = await _read_loaded_document(page, deadline) + if html is not None: + return html + # The page may have navigated while we polled, and the element found above would then + # belong to the document it left: reading it raises a stale-node error. Look it up + # again in whatever document is there now. + remaining = max(deadline - time.monotonic(), _PAGE_SOURCE_FALLBACK_FIND_SECONDS) + element = await page.find("html", timeout=remaining) return await element.get_html_async() +async def _read_loaded_document(page: Any, deadline: float) -> str | None: + """Return the document's HTML once it has finished parsing, or None by `deadline`. + + The document is read at least once, so a spent budget still returns a page that has + already loaded. + """ + errors = 0 + while True: + try: + html = await page.evaluate(_LOADED_DOCUMENT_EXPRESSION) + # An empty string means still loading. SeleniumBase's evaluate() returns None + # when CDP sends back no value. Only a non-empty string is a document. + if isinstance(html, str) and html: + return html + errors = 0 + except _CDP_OPERATION_ERRORS as e: + errors += 1 + logger.debug("Reading the loaded document failed (%d): %s", errors, e) + if errors >= _PAGE_LOAD_MAX_READ_ERRORS: + logger.warning( + "Could not read the loaded document after %d attempts; reading the page as it is", + errors, + ) + return None + if time.monotonic() >= deadline: + logger.warning("Page still loading after the page-source budget; reading it as it is") + return None + await asyncio.sleep(_PAGE_LOAD_POLL_SECONDS) + + async def _read_waiting_room_snapshot( page: Any, cancel_flag: Event | None ) -> _WaitingRoomSnapshot | None: diff --git a/tests/bypass/test_bypass_budgets.py b/tests/bypass/test_bypass_budgets.py index b7cabb9a..659f7dfe 100644 --- a/tests/bypass/test_bypass_budgets.py +++ b/tests/bypass/test_bypass_budgets.py @@ -268,24 +268,62 @@ def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, b assert attempts["n"] == 1 +_LOADED_DOCUMENT_EXPRESSION = ( + "document.readyState === 'loading' ? '' : document.documentElement.outerHTML" +) + + class _FakeElement: + def __init__(self, html: str = "solved") -> None: + self.html = html + async def get_html_async(self) -> str: - return "solved" + return self.html class _FakePage: - """A page that only produces its document after `ready_after` seconds of waiting.""" + """A page that only produces its document after `ready_after` seconds of waiting. - def __init__(self, ready_after: float = 0.0) -> None: + `states` scripts the document's readyState, one per read, the last one repeating. An + entry that is an exception is raised instead. A read while "loading" answers "", as + the real expression does; any other state answers `full`. The `html` element always + holds `partial`, so a test that gets `full` cannot have passed through the fallback. + The fixture removes sleeps, so the page progresses per read rather than per second. + Any other expression raises AssertionError, which the code under test does not catch, + so a typo in the expression fails the test instead of being absorbed. + """ + + def __init__( + self, + ready_after: float = 0.0, + *, + states: list[object] | None = None, + partial: str = "solved", + full: str = "solved", + ) -> None: self.ready_after = ready_after self.waited_with: list[float] = [] + self.states = states if states is not None else ["complete"] + self.partial = partial + self.full = full + self.state_polls = 0 async def find(self, selector: str, timeout: float = 1): self.waited_with.append(timeout) if timeout < self.ready_after: msg = f"Time ran out while waiting for: {{{selector}}}" raise TimeoutError(msg) - return _FakeElement() + return _FakeElement(self.partial) + + async def evaluate(self, expression: str) -> object: + if expression != _LOADED_DOCUMENT_EXPRESSION: + msg = f"unexpected expression: {expression}" + raise AssertionError(msg) + state = self.states[min(self.state_polls, len(self.states) - 1)] + self.state_polls += 1 + if isinstance(state, Exception): + raise state + return "" if state == "loading" else self.full def test_page_source_waits_longer_than_seleniumbases_one_second(bypass): @@ -313,3 +351,99 @@ def test_page_source_timeout_is_configurable(bypass, monkeypatch): asyncio.run(bypass._read_page_source(page)) assert page.waited_with == [45.0] + + +_PARTIAL_SEARCH_PAGE = "