Merge branch 'main' into renovate/fastapi-0.x

src/endpoints.py

@@ -26,6 +26,10 @@ router = APIRouter()
 
 BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
 
+CSP_HEADERS = frozenset(
+    {"content-security-policy", "content-security-policy-report-only"}
+)
+
 
 @router.get("/", include_in_schema=False)
 def read_root():
@@ -61,7 +65,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     request.url = request.url.replace('"', "").strip()
 
     if request.block_media:
-        async def block_media_route(route):
+        async def block_media_route(route) -> None:
             if route.request.resource_type in ("image", "media", "font"):
                 await route.abort()
             else:
@@ -69,6 +73,27 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
 
         await dep.page.route("**/*", block_media_route)
 
+    final_url: str | None = None
+
+    async def strip_csp_route(route) -> None:
+        nonlocal final_url
+        if route.request.resource_type != "document":
+            await route.continue_()
+            return
+        response = await route.fetch()
+        if route.request.frame == dep.page.main_frame:
+            final_url = response.url
+        await route.fulfill(
+            response=response,
+            headers={
+                key: value
+                for key, value in response.headers.items()
+                if key.lower() not in CSP_HEADERS
+            },
+        )
+
+    await dep.page.route("**/*", strip_csp_route)
+
     try:
         page_request = await dep.page.goto(
             request.url, timeout=timer.remaining() * 1000
@@ -93,14 +118,19 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
             status = HTTPStatus.OK
             logger.debug("Challenge solved successfully.")
         else:
-            await dep.page.wait_for_load_state(
-                "networkidle", timeout=timer.remaining() * 1000
-            )
+            try:
+                await dep.page.wait_for_load_state(
+                    "networkidle", timeout=timer.remaining() * 1000
+                )
+            except PlaywrightTimeoutError:
+                logger.info(
+                    "networkidle timed out after domcontentloaded; continuing with loaded page"
+                )
     except (TimeoutError, PlaywrightTimeoutError) as e:
-        logger.error("Timed out while solving the challenge")
+        logger.error("Timed out while loading the page or solving the challenge")
         raise HTTPException(
             status_code=408,
-            detail="Timed out while solving the challenge",
+            detail="Timed out while loading the page or solving the challenge",
         ) from e
 
     cookies = await dep.context.cookies()
@@ -130,7 +160,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
         message="Success",
         solution=Solution(
             user_agent=await dep.page.evaluate("navigator.userAgent"),
-            url=dep.page.url,
+            url=final_url if final_url is not None else dep.page.url,
             status=status,
             cookies=cookies,
             headers=page_request.headers if page_request else {},

src/models.py

@@ -5,11 +5,13 @@ from http.client import INTERNAL_SERVER_ERROR
 from typing import Any
 
 from playwright.sync_api import Cookie
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic.alias_generators import to_camel
 
 from src import consts
 
+MS_PER_SECOND = 1000
+
 
 class LinkRequest(BaseModel):
     model_config = {"populate_by_name": True}
@@ -21,7 +23,12 @@ class LinkRequest(BaseModel):
     url: str = Field(pattern=r"^https?://", default="https://")
     max_timeout: int = Field(
         default=60,
-        description="Maximum timeout in seconds for resolving the anti-bot challenge.",
+        alias="maxTimeout",
+        description=(
+            "Maximum timeout for resolving the anti-bot challenge. Values below 1000 "
+            "are treated as seconds; values of 1000 or more as milliseconds, matching "
+            "FlareSolverr's maxTimeout parameter."
+        ),
     )
     block_media: bool = Field(
         default=consts.BLOCK_MEDIA,
@@ -34,6 +41,14 @@ class LinkRequest(BaseModel):
         description="Return only cookies, skip the page HTML content in the response.",
     )
 
+    @field_validator("max_timeout")
+    @classmethod
+    def normalize_max_timeout(cls, value: int) -> int:
+        """Normalize FlareSolverr-style millisecond values to seconds."""
+        if value >= MS_PER_SECOND:
+            return value // MS_PER_SECOND
+        return value
+
 
 class HealthcheckResponse(BaseModel):
     model_config = {"alias_generator": to_camel, "populate_by_name": True}

tests/main_test.py

@@ -1,12 +1,17 @@
 from http import HTTPStatus
 from json import JSONDecodeError
+from unittest.mock import AsyncMock, MagicMock
 
 import httpx
 import pytest
+from fastapi import HTTPException
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
+from src.endpoints import read_item
 from src.models import LinkRequest
+from src.utils import BrowserDepClass
 
 client = TestClient(app)
 
@@ -82,3 +87,69 @@ def test_pdf_handling():
 
     decoded = base64.b64decode(solution["response"])
     assert decoded[:5] == b"%PDF-"
+
+
+@pytest.mark.parametrize(
+    ("payload", "expected"),
+    [
+        ({"max_timeout": 60}, 60),  # native API: seconds
+        ({"maxTimeout": 60}, 60),  # FlareSolverr alias, seconds-range value
+        ({"maxTimeout": 60000}, 60),  # FlareSolverr alias: milliseconds
+        ({"maxTimeout": 55000}, 55),
+        ({"maxTimeout": 1000}, 1),
+        ({}, 60),  # default
+    ],
+)
+def test_max_timeout_normalization(payload: dict, expected: int):
+    """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds."""
+    request = LinkRequest(url="https://example.com", **payload)
+    assert request.max_timeout == expected
+
+
+def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
+    """Build a browser dependency triple backed by mocks."""
+    page = AsyncMock()
+    page.url = "https://example.test/login"
+    page.goto.return_value = MagicMock(
+        status=HTTPStatus.OK, headers={"content-type": "text/html"}
+    )
+    page.title.return_value = "Login"
+    page.evaluate.return_value = "UnitTestBrowser/1.0"
+    page.content.return_value = "<html><title>Login</title></html>"
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        """Fail the wait when asked for a configured state."""
+        if state in (fail_states or set()):
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+
+    context = AsyncMock()
+    context.cookies.return_value = []
+    return BrowserDepClass(page=page, solver=AsyncMock(), context=context)
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_after_domcontentloaded_returns_content():
+    """Pages that never go idle after DOM load must still return their content."""
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        fake_dep(fail_states={"networkidle"}),
+    )
+
+    assert response.status == "ok"
+    assert response.solution.status == HTTPStatus.OK
+    assert response.solution.response == "<html><title>Login</title></html>"
+
+
+@pytest.mark.asyncio
+async def test_domcontentloaded_timeout_returns_408():
+    """Fatal timeouts during initial page load still return a controlled 408."""
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login"),
+            fake_dep(fail_states={"domcontentloaded"}),
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT