From 7b904a5ffdc88c1516b38d44287d48bf05c4cc29 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:27:54 +0200 Subject: [PATCH] feat: continue after networkidle timeout once domcontentloaded completes A page whose network never goes idle (background analytics, websockets) used to fail the whole request with a 408 once the networkidle wait expired. Since the DOM is fully usable after domcontentloaded, treat a networkidle timeout as non-fatal and return the loaded page instead. Fatal timeouts during initial load or challenge solving still return 408. Adds unit coverage for both paths using a fake page that fails configured load-state waits. --- src/endpoints.py | 26 ++++++++++++--- tests/main_test.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 46cbb68..c7d0afd 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -7,6 +7,7 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse +from playwright.async_api import Page from playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright_captcha import CaptchaType @@ -51,6 +52,23 @@ async def health_check(sb: BrowserDep): return HealthcheckResponse(user_agent=health_check_request.solution.user_agent) +async def _wait_for_networkidle(page: Page, timer: TimeoutTimer) -> None: + """ + Wait for the network to go idle; a timeout is non-fatal. + + Some sites keep background connections open (analytics beacons, + websockets, ...), so ``networkidle`` may never settle even though the page + is fully usable once ``domcontentloaded`` has fired. Log and continue + instead of failing the request. + """ + try: + await page.wait_for_load_state("networkidle", timeout=timer.remaining() * 1000) + except PlaywrightTimeoutError: + logger.info( + "networkidle timed out after domcontentloaded; continuing with loaded page" + ) + + @router.post("/v1") async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: """Handle POST requests.""" @@ -93,14 +111,12 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: status = HTTPStatus.OK logger.debug("Challenge solved successfully.") else: - await dep.page.wait_for_load_state( - "networkidle", timeout=timer.remaining() * 1000 - ) + await _wait_for_networkidle(dep.page, timer) except (TimeoutError, PlaywrightTimeoutError) as e: - logger.error("Timed out while solving the challenge") + logger.error("Timed out while loading the page or solving the challenge") raise HTTPException( status_code=408, - detail="Timed out while solving the challenge", + detail="Timed out while loading the page or solving the challenge", ) from e cookies = await dep.context.cookies() diff --git a/tests/main_test.py b/tests/main_test.py index d999007..ac9c74b 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,12 +1,17 @@ from http import HTTPStatus from json import JSONDecodeError +from types import SimpleNamespace import httpx import pytest +from fastapi import HTTPException +from playwright.async_api import TimeoutError as PlaywrightTimeoutError from starlette.testclient import TestClient from main import app +from src.endpoints import read_item from src.models import LinkRequest +from src.utils import BrowserDepClass client = TestClient(app) @@ -99,3 +104,79 @@ def test_max_timeout_normalization(payload: dict, expected: int): """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds.""" request = LinkRequest(url="https://example.com", **payload) assert request.max_timeout == expected + + +class FakeLoadStateError(PlaywrightTimeoutError): + """Raised by FakePage.wait_for_load_state for configured failing states.""" + + +class FakePage: + """Playwright Page double; load-state waits fail for configured states.""" + + url = "https://example.test/login" + + def __init__(self, *, fail_states: set[str] | None = None) -> None: + """Create a page whose waits fail for the given load states.""" + self.fail_states = fail_states or set() + + async def goto(self, _url: str, **_kwargs: object) -> SimpleNamespace: + """Return a successful navigation result.""" + return SimpleNamespace( + status=HTTPStatus.OK, headers={"content-type": "text/html"} + ) + + async def wait_for_load_state(self, state: str, **_kwargs: object) -> None: + """Fail for configured states; otherwise do nothing.""" + if state in self.fail_states: + message = "load state wait timed out" + raise FakeLoadStateError(message) + + async def title(self) -> str: + """Return a title that is not a challenge title.""" + return "Login" + + async def evaluate(self, _expression: str) -> str: + """Return the user agent the API reports.""" + return "UnitTestBrowser/1.0" + + async def content(self) -> str: + """Return the HTML body the API should return.""" + return "Login" + + +class FakeContext: + """Playwright BrowserContext double.""" + + async def cookies(self) -> list[object]: + """Return no cookies.""" + return [] + + +def make_dep(page: FakePage) -> BrowserDepClass: + """Build the browser dependency triple around a fake page.""" + return BrowserDepClass(page=page, solver=SimpleNamespace(), context=FakeContext()) + + +@pytest.mark.asyncio +async def test_networkidle_timeout_after_domcontentloaded_returns_content(): + """Pages that never go idle after DOM load must still return their content.""" + response = await read_item( + LinkRequest(url="https://example.test/login"), + make_dep(FakePage(fail_states={"networkidle"})), + ) + + assert response.status == "ok" + assert response.solution.status == HTTPStatus.OK + assert response.solution.response == "Login" + + +@pytest.mark.asyncio +async def test_domcontentloaded_timeout_returns_408(): + """Fatal timeouts during initial page load still return a controlled 408.""" + with pytest.raises(HTTPException) as exc: + await read_item( + LinkRequest(url="https://example.test/login"), + make_dep(FakePage(fail_states={"domcontentloaded"})), + ) + + assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT