Merge pull request #426 from ThePhaseless/fix/stale-snapshot-js-redirect

fix: read the page after the idle wait, not before it

src/content.py

@@ -10,9 +10,6 @@ async def build_response_content(
     page: Page,
     request: LinkRequest,
     page_request: object,
-    *,
-    challenge_detected: bool,
-    page_html: str | None,
 ) -> tuple[str, str]:
     """Build (content_type, response_content) from the settled page."""
     if request.return_only_cookies:
@@ -23,12 +20,7 @@ async def build_response_content(
     ):
         return await fetch_pdf_content(page)
 
-    response_content = (
-        page_html
-        if page_html is not None and not challenge_detected
-        else await page.content()
-    )
-    return "text/html", response_content
+    return "text/html", await page.content()
 
 
 async def fetch_pdf_content(page: Page) -> tuple[str, str]:

src/endpoints.py

@@ -66,9 +66,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     await setup_routes(request, dep)
 
     try:
-        challenge_detected, page_html, page_request = await _navigate_and_solve(
-            dep, request, timer
-        )
+        page_request = await _navigate_and_solve(dep, request, timer)
     except (TimeoutError, PlaywrightTimeoutError) as e:
         logger.error("Timed out while loading the page or solving the challenge")
         raise HTTPException(
@@ -90,8 +88,6 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
         dep.page,
         request,
         page_request,
-        challenge_detected=challenge_detected,
-        page_html=page_html,
     )
 
     user_agent = (
@@ -130,22 +126,17 @@ async def _navigate_and_solve(
     dep: BrowserDep,
     request: LinkRequest,
     timer: TimeoutTimer,
-) -> tuple[bool, str | None, object]:
-    """Navigate to the URL, then solve a challenge or wait for network idle."""
-    page_html: str | None = None
+) -> object:
+    """Navigate to the URL, solve a challenge if one is up, then wait for network idle."""
     page_request = await dep.page.goto(request.url, timeout=remaining_ms(timer))
     await dep.page.wait_for_load_state(
         state="domcontentloaded", timeout=remaining_ms(timer)
     )
 
-    if not await challenge_present(dep.page):
-        page_html = await dep.page.content()
-        await _wait_for_networkidle(dep, timer)
-        return False, page_html, page_request
-
-    await solve_challenge(dep.page, timer)
+    if await challenge_present(dep.page):
+        await solve_challenge(dep.page, timer)
     await _wait_for_networkidle(dep, timer)
-    return True, page_html, page_request
+    return page_request
 
 
 async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:

tests/main_test.py

@@ -206,6 +206,25 @@ async def test_networkidle_timeout_after_domcontentloaded_returns_content():
     assert response.solution.response == "<html><title>Login</title></html>"
 
 
+@pytest.mark.asyncio
+async def test_html_is_read_after_a_js_redirect_lands():
+    """A non-Cloudflare guard that redirects during the idle wait must not leak (#418)."""
+    dep = fake_dep()
+    dep.page.content.return_value = "<html><title>Checking</title></html>"
+    load_state = dep.page.wait_for_load_state.side_effect
+
+    def redirect_while_idling(state: str, **kwargs: object) -> None:
+        load_state(state, **kwargs)
+        if state == "networkidle":
+            dep.page.content.return_value = "<html><title>Article</title></html>"
+
+    dep.page.wait_for_load_state.side_effect = redirect_while_idling
+
+    response = await read_item(LinkRequest(url="https://example.test/login"), dep)
+
+    assert response.solution.response == "<html><title>Article</title></html>"
+
+
 @pytest.mark.asyncio
 async def test_domcontentloaded_timeout_returns_408():
     """Fatal timeouts during initial page load still return a controlled 408."""