diff --git a/src/endpoints.py b/src/endpoints.py index 46cbb68..c7d0afd 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -7,6 +7,7 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse +from playwright.async_api import Page from playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright_captcha import CaptchaType @@ -51,6 +52,23 @@ async def health_check(sb: BrowserDep): return HealthcheckResponse(user_agent=health_check_request.solution.user_agent) +async def _wait_for_networkidle(page: Page, timer: TimeoutTimer) -> None: + """ + Wait for the network to go idle; a timeout is non-fatal. + + Some sites keep background connections open (analytics beacons, + websockets, ...), so ``networkidle`` may never settle even though the page + is fully usable once ``domcontentloaded`` has fired. Log and continue + instead of failing the request. + """ + try: + await page.wait_for_load_state("networkidle", timeout=timer.remaining() * 1000) + except PlaywrightTimeoutError: + logger.info( + "networkidle timed out after domcontentloaded; continuing with loaded page" + ) + + @router.post("/v1") async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: """Handle POST requests.""" @@ -93,14 +111,12 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: status = HTTPStatus.OK logger.debug("Challenge solved successfully.") else: - await dep.page.wait_for_load_state( - "networkidle", timeout=timer.remaining() * 1000 - ) + await _wait_for_networkidle(dep.page, timer) except (TimeoutError, PlaywrightTimeoutError) as e: - logger.error("Timed out while solving the challenge") + logger.error("Timed out while loading the page or solving the challenge") raise HTTPException( status_code=408, - detail="Timed out while solving the challenge", + detail="Timed out while loading the page or solving the challenge", ) from e cookies = await dep.context.cookies() diff --git a/tests/main_test.py b/tests/main_test.py index d999007..ac9c74b 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,12 +1,17 @@ from http import HTTPStatus from json import JSONDecodeError +from types import SimpleNamespace import httpx import pytest +from fastapi import HTTPException +from playwright.async_api import TimeoutError as PlaywrightTimeoutError from starlette.testclient import TestClient from main import app +from src.endpoints import read_item from src.models import LinkRequest +from src.utils import BrowserDepClass client = TestClient(app) @@ -99,3 +104,79 @@ def test_max_timeout_normalization(payload: dict, expected: int): """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds.""" request = LinkRequest(url="https://example.com", **payload) assert request.max_timeout == expected + + +class FakeLoadStateError(PlaywrightTimeoutError): + """Raised by FakePage.wait_for_load_state for configured failing states.""" + + +class FakePage: + """Playwright Page double; load-state waits fail for configured states.""" + + url = "https://example.test/login" + + def __init__(self, *, fail_states: set[str] | None = None) -> None: + """Create a page whose waits fail for the given load states.""" + self.fail_states = fail_states or set() + + async def goto(self, _url: str, **_kwargs: object) -> SimpleNamespace: + """Return a successful navigation result.""" + return SimpleNamespace( + status=HTTPStatus.OK, headers={"content-type": "text/html"} + ) + + async def wait_for_load_state(self, state: str, **_kwargs: object) -> None: + """Fail for configured states; otherwise do nothing.""" + if state in self.fail_states: + message = "load state wait timed out" + raise FakeLoadStateError(message) + + async def title(self) -> str: + """Return a title that is not a challenge title.""" + return "Login" + + async def evaluate(self, _expression: str) -> str: + """Return the user agent the API reports.""" + return "UnitTestBrowser/1.0" + + async def content(self) -> str: + """Return the HTML body the API should return.""" + return "