diff --git a/src/content.py b/src/content.py index d5bc4cf..81a67d9 100644 --- a/src/content.py +++ b/src/content.py @@ -1,18 +1,18 @@ import base64 +from contextlib import suppress +from invisible_playwright.async_api import Error from playwright.async_api import Page from src.models import LinkRequest -from src.utils import logger +from src.utils import TimeoutTimer, logger, remaining_ms async def build_response_content( page: Page, request: LinkRequest, page_request: object, - *, - challenge_detected: bool, - page_html: str | None, + timer: TimeoutTimer, ) -> tuple[str, str]: """Build (content_type, response_content) from the settled page.""" if request.return_only_cookies: @@ -23,12 +23,18 @@ async def build_response_content( ): return await fetch_pdf_content(page) - response_content = ( - page_html - if page_html is not None and not challenge_detected - else await page.content() - ) - return "text/html", response_content + return "text/html", await read_settled_html(page, timer) + + +async def read_settled_html(page: Page, timer: TimeoutTimer) -> str: + """Read the page's HTML, reading it again if a navigation pulled it away.""" + try: + return await page.content() + except Error: + logger.info("Page navigated while being read; reading it again once idle") + with suppress(Error): + await page.wait_for_load_state("networkidle", timeout=remaining_ms(timer)) + return await page.content() async def fetch_pdf_content(page: Page) -> tuple[str, str]: diff --git a/src/endpoints.py b/src/endpoints.py index bd43dd6..d57c5f8 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -66,9 +66,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await setup_routes(request, dep) try: - challenge_detected, page_html, page_request = await _navigate_and_solve( - dep, request, timer - ) + page_request = await _navigate_and_solve(dep, request, timer) except (TimeoutError, PlaywrightTimeoutError) as e: logger.error("Timed out while loading the page or solving the challenge") raise HTTPException( @@ -90,8 +88,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: dep.page, request, page_request, - challenge_detected=challenge_detected, - page_html=page_html, + timer, ) user_agent = ( @@ -130,22 +127,17 @@ async def _navigate_and_solve( dep: BrowserDep, request: LinkRequest, timer: TimeoutTimer, -) -> tuple[bool, str | None, object]: - """Navigate to the URL, then solve a challenge or wait for network idle.""" - page_html: str | None = None +) -> object: + """Navigate to the URL, solve a challenge if one is up, then wait for network idle.""" page_request = await dep.page.goto(request.url, timeout=remaining_ms(timer)) await dep.page.wait_for_load_state( state="domcontentloaded", timeout=remaining_ms(timer) ) - if not await challenge_present(dep.page): - page_html = await dep.page.content() - await _wait_for_networkidle(dep, timer) - return False, page_html, page_request - - await solve_challenge(dep.page, timer) + if await challenge_present(dep.page): + await solve_challenge(dep.page, timer) await _wait_for_networkidle(dep, timer) - return True, page_html, page_request + return page_request async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None: diff --git a/tests/main_test.py b/tests/main_test.py index 06d0ac6..e152de8 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx2 import pytest from fastapi import HTTPException +from invisible_playwright.async_api import Error as InvisiblePlaywrightError from playwright.async_api import Error as PlaywrightError from playwright.async_api import TimeoutError as PlaywrightTimeoutError from starlette.testclient import TestClient @@ -203,6 +204,39 @@ async def test_networkidle_timeout_after_domcontentloaded_returns_content(): assert response.solution.response == "Login" +@pytest.mark.asyncio +async def test_html_is_read_after_a_js_redirect_lands(): + """A non-Cloudflare guard that redirects during the idle wait must not leak (#418).""" + dep = fake_dep() + dep.page.content.return_value = "Checking" + load_state = dep.page.wait_for_load_state.side_effect + + def redirect_while_idling(state: str, **kwargs: object) -> None: + load_state(state, **kwargs) + if state == "networkidle": + dep.page.content.return_value = "Article" + + dep.page.wait_for_load_state.side_effect = redirect_while_idling + + response = await read_item(LinkRequest(url="https://example.test/login"), dep) + + assert response.solution.response == "Article" + + +@pytest.mark.asyncio +async def test_html_read_racing_a_navigation_is_retried(): + """A read that loses its document to a navigation reads the new one instead.""" + dep = fake_dep() + dep.page.content.side_effect = [ + InvisiblePlaywrightError("Failed to find execution context with id = id-5"), + "Article", + ] + + response = await read_item(LinkRequest(url="https://example.test/login"), dep) + + assert response.solution.response == "Article" + + @pytest.mark.asyncio async def test_domcontentloaded_timeout_returns_408(): """Fatal timeouts during initial page load still return a controlled 408."""