diff --git a/src/endpoints.py b/src/endpoints.py index 46cbb68..5095874 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -26,6 +26,10 @@ router = APIRouter() BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)] +CSP_HEADERS = frozenset( + {"content-security-policy", "content-security-policy-report-only"} +) + @router.get("/", include_in_schema=False) def read_root(): @@ -61,7 +65,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: request.url = request.url.replace('"', "").strip() if request.block_media: - async def block_media_route(route): + async def block_media_route(route) -> None: if route.request.resource_type in ("image", "media", "font"): await route.abort() else: @@ -69,6 +73,40 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await dep.page.route("**/*", block_media_route) + # The Firefox engine evaluates JS via eval(), which pages whose CSP lacks + # 'unsafe-eval' block - every page.evaluate() then fails with "call to + # eval() blocked by CSP". Rewrite document responses without CSP headers + # so the user-agent read and captcha solver work on CSP-locked pages. + # Juggler only routes the first request of a redirect chain, so follow + # redirects inside the fetch and record the final URL ourselves. + final_url: str | None = None + + async def strip_csp_route(route) -> None: + nonlocal final_url + if route.request.resource_type != "document": + await route.continue_() + return + try: + response = await route.fetch() + except Exception: + await route.continue_() + return + if route.request.frame == dep.page.main_frame: + final_url = response.url + if response.headers.get("content-security-policy") or response.headers.get( + "content-security-policy-report-only" + ): + headers = { + key: value + for key, value in response.headers.items() + if key.lower() not in CSP_HEADERS + } + await route.fulfill(response=response, headers=headers) + else: + await route.fulfill(response=response) + + await dep.page.route("**/*", strip_csp_route) + try: page_request = await dep.page.goto( request.url, timeout=timer.remaining() * 1000 @@ -130,7 +168,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: message="Success", solution=Solution( user_agent=await dep.page.evaluate("navigator.userAgent"), - url=dep.page.url, + url=final_url if final_url is not None else dep.page.url, status=status, cookies=cookies, headers=page_request.headers if page_request else {},