From 69f6467ddec77bfce98bb9ba14825a2d6cf79942 Mon Sep 17 00:00:00 2001 From: Federico <85809106+feder-cr@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:00:31 +0200 Subject: [PATCH 1/7] deps: install invisible-playwright from PyPI instead of git --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 51a53b9..405b459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "API for getting cookies for Cloudflare challenges" readme = "README.md" dependencies = [ "fastapi[standard]==0.139.*", - "invisible_playwright @ git+https://github.com/feder-cr/invisible_playwright.git", + "invisible-playwright>=0.4.9", "playwright==1.60.*", "playwright-captcha==0.1.*", "pydantic==2.*", From 0b4d1a91ce25a0a5a5041b31f2a274a248e97456 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Fri, 7 Aug 2026 23:51:44 +0200 Subject: [PATCH 2/7] feat: accept FlareSolverr maxTimeout in milliseconds Add a maxTimeout alias to LinkRequest.max_timeout for FlareSolverr drop-in compatibility. Values of 1000 or more are treated as milliseconds and normalized to seconds; smaller values keep the native seconds semantics. Closes #382. --- src/models.py | 19 +++++++++++++++++-- tests/main_test.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/models.py b/src/models.py index baae038..61ed825 100644 --- a/src/models.py +++ b/src/models.py @@ -5,11 +5,13 @@ from http.client import INTERNAL_SERVER_ERROR from typing import Any from playwright.sync_api import Cookie -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from pydantic.alias_generators import to_camel from src import consts +MS_PER_SECOND = 1000 + class LinkRequest(BaseModel): model_config = {"populate_by_name": True} @@ -21,7 +23,12 @@ class LinkRequest(BaseModel): url: str = Field(pattern=r"^https?://", default="https://") max_timeout: int = Field( default=60, - description="Maximum timeout in seconds for resolving the anti-bot challenge.", + alias="maxTimeout", + description=( + "Maximum timeout for resolving the anti-bot challenge. Values below 1000 " + "are treated as seconds; values of 1000 or more as milliseconds, matching " + "FlareSolverr's maxTimeout parameter." + ), ) block_media: bool = Field( default=consts.BLOCK_MEDIA, @@ -34,6 +41,14 @@ class LinkRequest(BaseModel): description="Return only cookies, skip the page HTML content in the response.", ) + @field_validator("max_timeout") + @classmethod + def normalize_max_timeout(cls, value: int) -> int: + """Normalize FlareSolverr-style millisecond values to seconds.""" + if value >= MS_PER_SECOND: + return value // MS_PER_SECOND + return value + class HealthcheckResponse(BaseModel): model_config = {"alias_generator": to_camel, "populate_by_name": True} diff --git a/tests/main_test.py b/tests/main_test.py index 35e409d..d999007 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -82,3 +82,20 @@ def test_pdf_handling(): decoded = base64.b64decode(solution["response"]) assert decoded[:5] == b"%PDF-" + + +@pytest.mark.parametrize( + ("payload", "expected"), + [ + ({"max_timeout": 60}, 60), # native API: seconds + ({"maxTimeout": 60}, 60), # FlareSolverr alias, seconds-range value + ({"maxTimeout": 60000}, 60), # FlareSolverr alias: milliseconds + ({"maxTimeout": 55000}, 55), + ({"maxTimeout": 1000}, 1), + ({}, 60), # default + ], +) +def test_max_timeout_normalization(payload: dict, expected: int): + """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds.""" + request = LinkRequest(url="https://example.com", **payload) + assert request.max_timeout == expected From 6d447a0d67f12b0fb23bf779bdd3265780d3c0a5 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:25:46 +0200 Subject: [PATCH 3/7] fix: strip CSP headers from page responses so evaluate works The Firefox engine evaluates JS via eval(), which pages whose CSP lacks 'unsafe-eval' block - every page.evaluate() then fails with "call to eval() blocked by CSP". yggtorrent's search URL redirects to a page with such a CSP, crashing the user-agent read and 500ing /v1. Rewrite document responses without CSP headers via route.fetch + fulfill. Juggler only routes the first request of a redirect chain, so follow redirects inside the fetch and record the final URL ourselves instead of relying on page.url. --- src/endpoints.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 46cbb68..5095874 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -26,6 +26,10 @@ router = APIRouter() BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)] +CSP_HEADERS = frozenset( + {"content-security-policy", "content-security-policy-report-only"} +) + @router.get("/", include_in_schema=False) def read_root(): @@ -61,7 +65,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: request.url = request.url.replace('"', "").strip() if request.block_media: - async def block_media_route(route): + async def block_media_route(route) -> None: if route.request.resource_type in ("image", "media", "font"): await route.abort() else: @@ -69,6 +73,40 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await dep.page.route("**/*", block_media_route) + # The Firefox engine evaluates JS via eval(), which pages whose CSP lacks + # 'unsafe-eval' block - every page.evaluate() then fails with "call to + # eval() blocked by CSP". Rewrite document responses without CSP headers + # so the user-agent read and captcha solver work on CSP-locked pages. + # Juggler only routes the first request of a redirect chain, so follow + # redirects inside the fetch and record the final URL ourselves. + final_url: str | None = None + + async def strip_csp_route(route) -> None: + nonlocal final_url + if route.request.resource_type != "document": + await route.continue_() + return + try: + response = await route.fetch() + except Exception: + await route.continue_() + return + if route.request.frame == dep.page.main_frame: + final_url = response.url + if response.headers.get("content-security-policy") or response.headers.get( + "content-security-policy-report-only" + ): + headers = { + key: value + for key, value in response.headers.items() + if key.lower() not in CSP_HEADERS + } + await route.fulfill(response=response, headers=headers) + else: + await route.fulfill(response=response) + + await dep.page.route("**/*", strip_csp_route) + try: page_request = await dep.page.goto( request.url, timeout=timer.remaining() * 1000 @@ -130,7 +168,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: message="Success", solution=Solution( user_agent=await dep.page.evaluate("navigator.userAgent"), - url=dep.page.url, + url=final_url if final_url is not None else dep.page.url, status=status, cookies=cookies, headers=page_request.headers if page_request else {}, From 7b904a5ffdc88c1516b38d44287d48bf05c4cc29 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:27:54 +0200 Subject: [PATCH 4/7] feat: continue after networkidle timeout once domcontentloaded completes A page whose network never goes idle (background analytics, websockets) used to fail the whole request with a 408 once the networkidle wait expired. Since the DOM is fully usable after domcontentloaded, treat a networkidle timeout as non-fatal and return the loaded page instead. Fatal timeouts during initial load or challenge solving still return 408. Adds unit coverage for both paths using a fake page that fails configured load-state waits. --- src/endpoints.py | 26 ++++++++++++--- tests/main_test.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 46cbb68..c7d0afd 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -7,6 +7,7 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse +from playwright.async_api import Page from playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright_captcha import CaptchaType @@ -51,6 +52,23 @@ async def health_check(sb: BrowserDep): return HealthcheckResponse(user_agent=health_check_request.solution.user_agent) +async def _wait_for_networkidle(page: Page, timer: TimeoutTimer) -> None: + """ + Wait for the network to go idle; a timeout is non-fatal. + + Some sites keep background connections open (analytics beacons, + websockets, ...), so ``networkidle`` may never settle even though the page + is fully usable once ``domcontentloaded`` has fired. Log and continue + instead of failing the request. + """ + try: + await page.wait_for_load_state("networkidle", timeout=timer.remaining() * 1000) + except PlaywrightTimeoutError: + logger.info( + "networkidle timed out after domcontentloaded; continuing with loaded page" + ) + + @router.post("/v1") async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: """Handle POST requests.""" @@ -93,14 +111,12 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: status = HTTPStatus.OK logger.debug("Challenge solved successfully.") else: - await dep.page.wait_for_load_state( - "networkidle", timeout=timer.remaining() * 1000 - ) + await _wait_for_networkidle(dep.page, timer) except (TimeoutError, PlaywrightTimeoutError) as e: - logger.error("Timed out while solving the challenge") + logger.error("Timed out while loading the page or solving the challenge") raise HTTPException( status_code=408, - detail="Timed out while solving the challenge", + detail="Timed out while loading the page or solving the challenge", ) from e cookies = await dep.context.cookies() diff --git a/tests/main_test.py b/tests/main_test.py index d999007..ac9c74b 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,12 +1,17 @@ from http import HTTPStatus from json import JSONDecodeError +from types import SimpleNamespace import httpx import pytest +from fastapi import HTTPException +from playwright.async_api import TimeoutError as PlaywrightTimeoutError from starlette.testclient import TestClient from main import app +from src.endpoints import read_item from src.models import LinkRequest +from src.utils import BrowserDepClass client = TestClient(app) @@ -99,3 +104,79 @@ def test_max_timeout_normalization(payload: dict, expected: int): """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds.""" request = LinkRequest(url="https://example.com", **payload) assert request.max_timeout == expected + + +class FakeLoadStateError(PlaywrightTimeoutError): + """Raised by FakePage.wait_for_load_state for configured failing states.""" + + +class FakePage: + """Playwright Page double; load-state waits fail for configured states.""" + + url = "https://example.test/login" + + def __init__(self, *, fail_states: set[str] | None = None) -> None: + """Create a page whose waits fail for the given load states.""" + self.fail_states = fail_states or set() + + async def goto(self, _url: str, **_kwargs: object) -> SimpleNamespace: + """Return a successful navigation result.""" + return SimpleNamespace( + status=HTTPStatus.OK, headers={"content-type": "text/html"} + ) + + async def wait_for_load_state(self, state: str, **_kwargs: object) -> None: + """Fail for configured states; otherwise do nothing.""" + if state in self.fail_states: + message = "load state wait timed out" + raise FakeLoadStateError(message) + + async def title(self) -> str: + """Return a title that is not a challenge title.""" + return "Login" + + async def evaluate(self, _expression: str) -> str: + """Return the user agent the API reports.""" + return "UnitTestBrowser/1.0" + + async def content(self) -> str: + """Return the HTML body the API should return.""" + return "Login" + + +class FakeContext: + """Playwright BrowserContext double.""" + + async def cookies(self) -> list[object]: + """Return no cookies.""" + return [] + + +def make_dep(page: FakePage) -> BrowserDepClass: + """Build the browser dependency triple around a fake page.""" + return BrowserDepClass(page=page, solver=SimpleNamespace(), context=FakeContext()) + + +@pytest.mark.asyncio +async def test_networkidle_timeout_after_domcontentloaded_returns_content(): + """Pages that never go idle after DOM load must still return their content.""" + response = await read_item( + LinkRequest(url="https://example.test/login"), + make_dep(FakePage(fail_states={"networkidle"})), + ) + + assert response.status == "ok" + assert response.solution.status == HTTPStatus.OK + assert response.solution.response == "Login" + + +@pytest.mark.asyncio +async def test_domcontentloaded_timeout_returns_408(): + """Fatal timeouts during initial page load still return a controlled 408.""" + with pytest.raises(HTTPException) as exc: + await read_item( + LinkRequest(url="https://example.test/login"), + make_dep(FakePage(fail_states={"domcontentloaded"})), + ) + + assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT From e2fd2e6d42835dff4ec8dfb39ee0ccf43c1da173 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:31:25 +0200 Subject: [PATCH 5/7] refactor: simplify CSP stripping handler Drop error handling and conditional branches that duplicated the pass-through path; rely on the goto timeout as before. --- src/endpoints.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 5095874..5470e9e 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -73,12 +73,6 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await dep.page.route("**/*", block_media_route) - # The Firefox engine evaluates JS via eval(), which pages whose CSP lacks - # 'unsafe-eval' block - every page.evaluate() then fails with "call to - # eval() blocked by CSP". Rewrite document responses without CSP headers - # so the user-agent read and captcha solver work on CSP-locked pages. - # Juggler only routes the first request of a redirect chain, so follow - # redirects inside the fetch and record the final URL ourselves. final_url: str | None = None async def strip_csp_route(route) -> None: @@ -86,24 +80,17 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: if route.request.resource_type != "document": await route.continue_() return - try: - response = await route.fetch() - except Exception: - await route.continue_() - return + response = await route.fetch() if route.request.frame == dep.page.main_frame: final_url = response.url - if response.headers.get("content-security-policy") or response.headers.get( - "content-security-policy-report-only" - ): - headers = { + await route.fulfill( + response=response, + headers={ key: value for key, value in response.headers.items() if key.lower() not in CSP_HEADERS - } - await route.fulfill(response=response, headers=headers) - else: - await route.fulfill(response=response) + }, + ) await dep.page.route("**/*", strip_csp_route) From 490e2fad97a5c70228deb9bdfc98d8f384a00263 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:37:09 +0200 Subject: [PATCH 6/7] refactor: keep networkidle timeout handling inline, mock-based tests --- src/endpoints.py | 31 ++++++++------------- tests/main_test.py | 69 ++++++++++++++-------------------------------- 2 files changed, 33 insertions(+), 67 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index c7d0afd..49a9b29 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -7,7 +7,6 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse -from playwright.async_api import Page from playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright_captcha import CaptchaType @@ -52,23 +51,6 @@ async def health_check(sb: BrowserDep): return HealthcheckResponse(user_agent=health_check_request.solution.user_agent) -async def _wait_for_networkidle(page: Page, timer: TimeoutTimer) -> None: - """ - Wait for the network to go idle; a timeout is non-fatal. - - Some sites keep background connections open (analytics beacons, - websockets, ...), so ``networkidle`` may never settle even though the page - is fully usable once ``domcontentloaded`` has fired. Log and continue - instead of failing the request. - """ - try: - await page.wait_for_load_state("networkidle", timeout=timer.remaining() * 1000) - except PlaywrightTimeoutError: - logger.info( - "networkidle timed out after domcontentloaded; continuing with loaded page" - ) - - @router.post("/v1") async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: """Handle POST requests.""" @@ -111,7 +93,18 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: status = HTTPStatus.OK logger.debug("Challenge solved successfully.") else: - await _wait_for_networkidle(dep.page, timer) + # Best-effort: some sites keep background connections open + # (analytics beacons, websockets, ...), so ``networkidle`` may + # never settle. The page is fully usable once ``domcontentloaded`` + # has fired, so log and continue instead of failing the request. + try: + await dep.page.wait_for_load_state( + "networkidle", timeout=timer.remaining() * 1000 + ) + except PlaywrightTimeoutError: + logger.info( + "networkidle timed out after domcontentloaded; continuing with loaded page" + ) except (TimeoutError, PlaywrightTimeoutError) as e: logger.error("Timed out while loading the page or solving the challenge") raise HTTPException( diff --git a/tests/main_test.py b/tests/main_test.py index ac9c74b..df512db 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,6 +1,6 @@ from http import HTTPStatus from json import JSONDecodeError -from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock import httpx import pytest @@ -106,55 +106,28 @@ def test_max_timeout_normalization(payload: dict, expected: int): assert request.max_timeout == expected -class FakeLoadStateError(PlaywrightTimeoutError): - """Raised by FakePage.wait_for_load_state for configured failing states.""" +def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass: + """Build a browser dependency triple backed by mocks.""" + page = AsyncMock() + page.url = "https://example.test/login" + page.goto.return_value = MagicMock( + status=HTTPStatus.OK, headers={"content-type": "text/html"} + ) + page.title.return_value = "Login" + page.evaluate.return_value = "UnitTestBrowser/1.0" + page.content.return_value = "Login" - -class FakePage: - """Playwright Page double; load-state waits fail for configured states.""" - - url = "https://example.test/login" - - def __init__(self, *, fail_states: set[str] | None = None) -> None: - """Create a page whose waits fail for the given load states.""" - self.fail_states = fail_states or set() - - async def goto(self, _url: str, **_kwargs: object) -> SimpleNamespace: - """Return a successful navigation result.""" - return SimpleNamespace( - status=HTTPStatus.OK, headers={"content-type": "text/html"} - ) - - async def wait_for_load_state(self, state: str, **_kwargs: object) -> None: - """Fail for configured states; otherwise do nothing.""" - if state in self.fail_states: + def wait_for_load_state(state: str, **_kwargs: object) -> None: + """Fail the wait when asked for a configured state.""" + if state in (fail_states or set()): message = "load state wait timed out" - raise FakeLoadStateError(message) + raise PlaywrightTimeoutError(message) - async def title(self) -> str: - """Return a title that is not a challenge title.""" - return "Login" + page.wait_for_load_state.side_effect = wait_for_load_state - async def evaluate(self, _expression: str) -> str: - """Return the user agent the API reports.""" - return "UnitTestBrowser/1.0" - - async def content(self) -> str: - """Return the HTML body the API should return.""" - return "Login" - - -class FakeContext: - """Playwright BrowserContext double.""" - - async def cookies(self) -> list[object]: - """Return no cookies.""" - return [] - - -def make_dep(page: FakePage) -> BrowserDepClass: - """Build the browser dependency triple around a fake page.""" - return BrowserDepClass(page=page, solver=SimpleNamespace(), context=FakeContext()) + context = AsyncMock() + context.cookies.return_value = [] + return BrowserDepClass(page=page, solver=AsyncMock(), context=context) @pytest.mark.asyncio @@ -162,7 +135,7 @@ async def test_networkidle_timeout_after_domcontentloaded_returns_content(): """Pages that never go idle after DOM load must still return their content.""" response = await read_item( LinkRequest(url="https://example.test/login"), - make_dep(FakePage(fail_states={"networkidle"})), + fake_dep(fail_states={"networkidle"}), ) assert response.status == "ok" @@ -176,7 +149,7 @@ async def test_domcontentloaded_timeout_returns_408(): with pytest.raises(HTTPException) as exc: await read_item( LinkRequest(url="https://example.test/login"), - make_dep(FakePage(fail_states={"domcontentloaded"})), + fake_dep(fail_states={"domcontentloaded"}), ) assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT From 5c4d0393b4aec91ac5befb757db61d64b1c1e805 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:49:56 +0200 Subject: [PATCH 7/7] chore: drop redundant comment on networkidle best-effort wait --- src/endpoints.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index ca019c5..dbe6788 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -118,10 +118,6 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: status = HTTPStatus.OK logger.debug("Challenge solved successfully.") else: - # Best-effort: some sites keep background connections open - # (analytics beacons, websockets, ...), so ``networkidle`` may - # never settle. The page is fully usable once ``domcontentloaded`` - # has fired, so log and continue instead of failing the request. try: await dep.page.wait_for_load_state( "networkidle", timeout=timer.remaining() * 1000