refactor: read the settled page once, without a retry

The retry in read_settled_html covered page.content() racing a
navigation. #418 only showed that race on the early read taken right
after domcontentloaded, and this branch already removes that read. The
late read now matches the Cloudflare path, which reads after the idle
wait and has never retried.

When the retry fires at the end of the budget, remaining_ms() is 1 ms.
The second idle wait then times out at once, and the second read lands
in the same 60-150 ms window that failed. So the retry adds a helper and
threads timer into content.py for a case it does not cover.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ThePhaselessandClaude Opus 5.5 committed 2026-10-09 00:03:10 +02:00
1 parent 9e6762b053
commit 80dc9fc272
3 files changed
+2 -31

No files matched your search

+2 -16
View File
@@ -1,18 +1,15 @@
import base64
from contextlib import suppress
from invisible_playwright.async_api import Error
from playwright.async_api import Page
from src.models import LinkRequest
from src.utils import TimeoutTimer, logger, remaining_ms
from src.utils import logger
async def build_response_content(
page: Page,
request: LinkRequest,
page_request: object,
timer: TimeoutTimer,
) -> tuple[str, str]:
"""Build (content_type, response_content) from the settled page."""
if request.return_only_cookies:
@@ -23,18 +20,7 @@ async def build_response_content(
):
return await fetch_pdf_content(page)
return "text/html", await read_settled_html(page, timer)
async def read_settled_html(page: Page, timer: TimeoutTimer) -> str:
"""Read the page's HTML, reading it again if a navigation pulled it away."""
try:
return await page.content()
except Error:
logger.info("Page navigated while being read; reading it again once idle")
with suppress(Error):
await page.wait_for_load_state("networkidle", timeout=remaining_ms(timer))
return await page.content()
return "text/html", await page.content()
async def fetch_pdf_content(page: Page) -> tuple[str, str]:
-1
View File
@@ -88,7 +88,6 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
dep.page,
request,
page_request,
timer,
)
user_agent = (
-14
View File
@@ -225,20 +225,6 @@ async def test_html_is_read_after_a_js_redirect_lands():
assert response.solution.response == "<html><title>Article</title></html>"
@pytest.mark.asyncio
async def test_html_read_racing_a_navigation_is_retried():
"""A read that loses its document to a navigation reads the new one instead."""
dep = fake_dep()
dep.page.content.side_effect = [
PlaywrightError("Failed to find execution context with id = id-5"),
"<html><title>Article</title></html>",
]
response = await read_item(LinkRequest(url="https://example.test/login"), dep)
assert response.solution.response == "<html><title>Article</title></html>"
@pytest.mark.asyncio
async def test_domcontentloaded_timeout_returns_408():
"""Fatal timeouts during initial page load still return a controlled 408."""