diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index c4214a8f..dcbb994d 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -91,6 +91,22 @@ _HELPER_IDLE_TIMEOUT_DEFAULT = 180.0 # How long to wait for a solved page to produce its document before the attempt is # abandoned. SeleniumBase's own get_page_source() allows one second; see _read_page_source. _PAGE_SOURCE_TIMEOUT_DEFAULT = 20.0 +# How often to ask whether a solved page has finished parsing, inside that same budget. +_PAGE_LOAD_POLL_SECONDS = 0.25 +# Consecutive failed reads of the document state before falling back to the element read. +# A page whose CDP calls keep failing will not start answering, and every retry spends +# budget the caller needs. +_PAGE_LOAD_MAX_READ_ERRORS = 3 +# The least time the fallback gets to look the `html` element up again, even when the +# budget is spent: enough for one real lookup on a page that is there. It matches +# SeleniumBase's own default for find(). +_PAGE_SOURCE_FALLBACK_FIND_SECONDS = 1.0 +# One CDP call, not two: a navigation can commit between a readiness read and an HTML +# read, so the old document would pass the check and the first chunk of the new one would +# be returned. Same reasoning as the atomic snapshot in _wait_for_aa_download_page. +_LOADED_DOCUMENT_EXPRESSION = ( + "document.readyState === 'loading' ? '' : document.documentElement.outerHTML" +) # Leave time inside the existing browser watchdog for challenge solving and cleanup. _AA_WAITING_ROOM_TIMEOUT_SECONDS = 300.0 _AA_WAITING_ROOM_POLL_SECONDS = 1.0 @@ -855,15 +871,62 @@ async def _read_page_source(page: Any) -> str: challenge is often still navigating to the real content, so the read times out even though the solve succeeded: the whole attempt is retried, and the repeated requests are what earn a 429 from a host that was about to serve us. + + The `html` element existing is not enough either: it is there as soon as the first + bytes are parsed. Anna's Archive streams the head and filters of a search page before + the results, and the bypass check passes on that prefix, so reading the element at + once returned about 120 KB of a 745 KB page with no results table in it. So within the + same budget, wait for `document.readyState` to leave "loading" and read the whole + document. If the budget runs out or the reads keep failing, look the `html` element + up again and return what it holds, which is what this returned before. """ timeout = _coerce_non_negative_float( app_config.get("BYPASS_PAGE_SOURCE_TIMEOUT", _PAGE_SOURCE_TIMEOUT_DEFAULT), _PAGE_SOURCE_TIMEOUT_DEFAULT, ) + deadline = time.monotonic() + timeout element = await page.find("html", timeout=timeout) + html = await _read_loaded_document(page, deadline) + if html is not None: + return html + # The page may have navigated while we polled, and the element found above would then + # belong to the document it left: reading it raises a stale-node error. Look it up + # again in whatever document is there now. + remaining = max(deadline - time.monotonic(), _PAGE_SOURCE_FALLBACK_FIND_SECONDS) + element = await page.find("html", timeout=remaining) return await element.get_html_async() +async def _read_loaded_document(page: Any, deadline: float) -> str | None: + """Return the document's HTML once it has finished parsing, or None by `deadline`. + + The document is read at least once, so a spent budget still returns a page that has + already loaded. + """ + errors = 0 + while True: + try: + html = await page.evaluate(_LOADED_DOCUMENT_EXPRESSION) + # An empty string means still loading. SeleniumBase's evaluate() returns None + # when CDP sends back no value. Only a non-empty string is a document. + if isinstance(html, str) and html: + return html + errors = 0 + except _CDP_OPERATION_ERRORS as e: + errors += 1 + logger.debug("Reading the loaded document failed (%d): %s", errors, e) + if errors >= _PAGE_LOAD_MAX_READ_ERRORS: + logger.warning( + "Could not read the loaded document after %d attempts; reading the page as it is", + errors, + ) + return None + if time.monotonic() >= deadline: + logger.warning("Page still loading after the page-source budget; reading it as it is") + return None + await asyncio.sleep(_PAGE_LOAD_POLL_SECONDS) + + async def _read_waiting_room_snapshot( page: Any, cancel_flag: Event | None ) -> _WaitingRoomSnapshot | None: diff --git a/tests/bypass/test_bypass_budgets.py b/tests/bypass/test_bypass_budgets.py index b7cabb9a..659f7dfe 100644 --- a/tests/bypass/test_bypass_budgets.py +++ b/tests/bypass/test_bypass_budgets.py @@ -268,24 +268,62 @@ def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, b assert attempts["n"] == 1 +_LOADED_DOCUMENT_EXPRESSION = ( + "document.readyState === 'loading' ? '' : document.documentElement.outerHTML" +) + + class _FakeElement: + def __init__(self, html: str = "solved") -> None: + self.html = html + async def get_html_async(self) -> str: - return "solved" + return self.html class _FakePage: - """A page that only produces its document after `ready_after` seconds of waiting.""" + """A page that only produces its document after `ready_after` seconds of waiting. - def __init__(self, ready_after: float = 0.0) -> None: + `states` scripts the document's readyState, one per read, the last one repeating. An + entry that is an exception is raised instead. A read while "loading" answers "", as + the real expression does; any other state answers `full`. The `html` element always + holds `partial`, so a test that gets `full` cannot have passed through the fallback. + The fixture removes sleeps, so the page progresses per read rather than per second. + Any other expression raises AssertionError, which the code under test does not catch, + so a typo in the expression fails the test instead of being absorbed. + """ + + def __init__( + self, + ready_after: float = 0.0, + *, + states: list[object] | None = None, + partial: str = "solved", + full: str = "solved", + ) -> None: self.ready_after = ready_after self.waited_with: list[float] = [] + self.states = states if states is not None else ["complete"] + self.partial = partial + self.full = full + self.state_polls = 0 async def find(self, selector: str, timeout: float = 1): self.waited_with.append(timeout) if timeout < self.ready_after: msg = f"Time ran out while waiting for: {{{selector}}}" raise TimeoutError(msg) - return _FakeElement() + return _FakeElement(self.partial) + + async def evaluate(self, expression: str) -> object: + if expression != _LOADED_DOCUMENT_EXPRESSION: + msg = f"unexpected expression: {expression}" + raise AssertionError(msg) + state = self.states[min(self.state_polls, len(self.states) - 1)] + self.state_polls += 1 + if isinstance(state, Exception): + raise state + return "" if state == "loading" else self.full def test_page_source_waits_longer_than_seleniumbases_one_second(bypass): @@ -313,3 +351,99 @@ def test_page_source_timeout_is_configurable(bypass, monkeypatch): asyncio.run(bypass._read_page_source(page)) assert page.waited_with == [45.0] + + +_PARTIAL_SEARCH_PAGE = "Search
filters" +_FULL_SEARCH_PAGE = _PARTIAL_SEARCH_PAGE + "results
" + + +def test_page_source_waits_for_the_document_to_finish_loading(bypass): + """The `html` element exists before the page has arrived. + + On 1.7.0 a cleared DDoS-Guard check returned 122,986 bytes of an Anna's Archive search + page of about 745,000, read while the results were still streaming in, and the user + was shown "No results found". + """ + page = _FakePage( + states=["loading", "loading", "loading", "interactive"], + partial=_PARTIAL_SEARCH_PAGE, + full=_FULL_SEARCH_PAGE, + ) + + assert asyncio.run(bypass._read_page_source(page)) == _FULL_SEARCH_PAGE + assert page.state_polls == 4 + + +def test_page_source_falls_back_to_the_element_when_loading_outlasts_the_budget( + monkeypatch, bypass +): + """A page that never finishes loading must be read as before, not raise or hang.""" + clock = {"now": 0.0} + monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"]) + page = _FakePage(states=["loading"], partial=_PARTIAL_SEARCH_PAGE, full=_FULL_SEARCH_PAGE) + original_evaluate = page.evaluate + + async def _evaluate_slowly(expression: str) -> object: + clock["now"] += 1.0 + return await original_evaluate(expression) + + page.evaluate = _evaluate_slowly + + assert asyncio.run(bypass._read_page_source(page)) == _PARTIAL_SEARCH_PAGE + assert clock["now"] >= bypass._PAGE_SOURCE_TIMEOUT_DEFAULT + assert page.state_polls <= bypass._PAGE_SOURCE_TIMEOUT_DEFAULT + 1 + # The budget is spent, so the second lookup gets the floor, not a negative timeout. + assert page.waited_with == [ + bypass._PAGE_SOURCE_TIMEOUT_DEFAULT, + bypass._PAGE_SOURCE_FALLBACK_FIND_SECONDS, + ] + + +class _StaleNodeElement: + """An element of a document the page has left: CDP no longer knows its node.""" + + async def get_html_async(self) -> str: + from seleniumbase.undetected.cdp_driver.connection import ProtocolException + + raise ProtocolException("No node with given id found") + + +def test_page_source_fallback_reads_the_document_the_page_is_on_now(monkeypatch, bypass): + """A navigation during the wait leaves the first `html` element stale. + + Reading it raised a CDP error out of _read_page_source and failed the attempt, when + the page had a document to return. + """ + clock = {"now": 0.0} + monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"]) + page = _FakePage(states=["loading"], partial=_PARTIAL_SEARCH_PAGE, full=_FULL_SEARCH_PAGE) + original_evaluate = page.evaluate + elements: list[object] = [_StaleNodeElement(), _FakeElement("new page")] + + async def _evaluate_slowly(expression: str) -> object: + clock["now"] += 1.0 + return await original_evaluate(expression) + + async def _find(selector: str, timeout: float = 1) -> object: + page.waited_with.append(timeout) + return elements.pop(0) + + page.evaluate = _evaluate_slowly + page.find = _find + + assert asyncio.run(bypass._read_page_source(page)) == "new page" + assert elements == [] + + +def test_page_source_survives_a_cdp_error_while_waiting(bypass): + """One failed state read must not drop the page back to its half-loaded prefix.""" + from seleniumbase.undetected.cdp_driver.connection import ProtocolException + + page = _FakePage( + states=[ProtocolException("Execution context was destroyed"), "loading", "complete"], + partial=_PARTIAL_SEARCH_PAGE, + full=_FULL_SEARCH_PAGE, + ) + + assert asyncio.run(bypass._read_page_source(page)) == _FULL_SEARCH_PAGE + assert page.state_polls == 3