feat: continue after networkidle timeout once domcontentloaded completes

A page whose network never goes idle (background analytics, websockets)
used to fail the whole request with a 408 once the networkidle wait
expired. Since the DOM is fully usable after domcontentloaded, treat a
networkidle timeout as non-fatal and return the loaded page instead.
Fatal timeouts during initial load or challenge solving still return 408.

Adds unit coverage for both paths using a fake page that fails
configured load-state waits.

src/endpoints.py

@@ -7,6 +7,7 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
+from playwright.async_api import Page
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright_captcha import CaptchaType
 
@@ -51,6 +52,23 @@ async def health_check(sb: BrowserDep):
     return HealthcheckResponse(user_agent=health_check_request.solution.user_agent)
 
 
+async def _wait_for_networkidle(page: Page, timer: TimeoutTimer) -> None:
+    """
+    Wait for the network to go idle; a timeout is non-fatal.
+
+    Some sites keep background connections open (analytics beacons,
+    websockets, ...), so ``networkidle`` may never settle even though the page
+    is fully usable once ``domcontentloaded`` has fired. Log and continue
+    instead of failing the request.
+    """
+    try:
+        await page.wait_for_load_state("networkidle", timeout=timer.remaining() * 1000)
+    except PlaywrightTimeoutError:
+        logger.info(
+            "networkidle timed out after domcontentloaded; continuing with loaded page"
+        )
+
+
 @router.post("/v1")
 async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     """Handle POST requests."""
@@ -93,14 +111,12 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
             status = HTTPStatus.OK
             logger.debug("Challenge solved successfully.")
         else:
-            await dep.page.wait_for_load_state(
-                "networkidle", timeout=timer.remaining() * 1000
-            )
+            await _wait_for_networkidle(dep.page, timer)
     except (TimeoutError, PlaywrightTimeoutError) as e:
-        logger.error("Timed out while solving the challenge")
+        logger.error("Timed out while loading the page or solving the challenge")
         raise HTTPException(
             status_code=408,
-            detail="Timed out while solving the challenge",
+            detail="Timed out while loading the page or solving the challenge",
         ) from e
 
     cookies = await dep.context.cookies()

tests/main_test.py

@@ -1,12 +1,17 @@
 from http import HTTPStatus
 from json import JSONDecodeError
+from types import SimpleNamespace
 
 import httpx
 import pytest
+from fastapi import HTTPException
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
+from src.endpoints import read_item
 from src.models import LinkRequest
+from src.utils import BrowserDepClass
 
 client = TestClient(app)
 
@@ -99,3 +104,79 @@ def test_max_timeout_normalization(payload: dict, expected: int):
     """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds."""
     request = LinkRequest(url="https://example.com", **payload)
     assert request.max_timeout == expected
+
+
+class FakeLoadStateError(PlaywrightTimeoutError):
+    """Raised by FakePage.wait_for_load_state for configured failing states."""
+
+
+class FakePage:
+    """Playwright Page double; load-state waits fail for configured states."""
+
+    url = "https://example.test/login"
+
+    def __init__(self, *, fail_states: set[str] | None = None) -> None:
+        """Create a page whose waits fail for the given load states."""
+        self.fail_states = fail_states or set()
+
+    async def goto(self, _url: str, **_kwargs: object) -> SimpleNamespace:
+        """Return a successful navigation result."""
+        return SimpleNamespace(
+            status=HTTPStatus.OK, headers={"content-type": "text/html"}
+        )
+
+    async def wait_for_load_state(self, state: str, **_kwargs: object) -> None:
+        """Fail for configured states; otherwise do nothing."""
+        if state in self.fail_states:
+            message = "load state wait timed out"
+            raise FakeLoadStateError(message)
+
+    async def title(self) -> str:
+        """Return a title that is not a challenge title."""
+        return "Login"
+
+    async def evaluate(self, _expression: str) -> str:
+        """Return the user agent the API reports."""
+        return "UnitTestBrowser/1.0"
+
+    async def content(self) -> str:
+        """Return the HTML body the API should return."""
+        return "<html><title>Login</title></html>"
+
+
+class FakeContext:
+    """Playwright BrowserContext double."""
+
+    async def cookies(self) -> list[object]:
+        """Return no cookies."""
+        return []
+
+
+def make_dep(page: FakePage) -> BrowserDepClass:
+    """Build the browser dependency triple around a fake page."""
+    return BrowserDepClass(page=page, solver=SimpleNamespace(), context=FakeContext())
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_after_domcontentloaded_returns_content():
+    """Pages that never go idle after DOM load must still return their content."""
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        make_dep(FakePage(fail_states={"networkidle"})),
+    )
+
+    assert response.status == "ok"
+    assert response.solution.status == HTTPStatus.OK
+    assert response.solution.response == "<html><title>Login</title></html>"
+
+
+@pytest.mark.asyncio
+async def test_domcontentloaded_timeout_returns_408():
+    """Fatal timeouts during initial page load still return a controlled 408."""
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login"),
+            make_dep(FakePage(fail_states={"domcontentloaded"})),
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT