From 6d447a0d67f12b0fb23bf779bdd3265780d3c0a5 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:25:46 +0200 Subject: [PATCH 1/2] fix: strip CSP headers from page responses so evaluate works The Firefox engine evaluates JS via eval(), which pages whose CSP lacks 'unsafe-eval' block - every page.evaluate() then fails with "call to eval() blocked by CSP". yggtorrent's search URL redirects to a page with such a CSP, crashing the user-agent read and 500ing /v1. Rewrite document responses without CSP headers via route.fetch + fulfill. Juggler only routes the first request of a redirect chain, so follow redirects inside the fetch and record the final URL ourselves instead of relying on page.url. --- src/endpoints.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 46cbb68..5095874 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -26,6 +26,10 @@ router = APIRouter() BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)] +CSP_HEADERS = frozenset( + {"content-security-policy", "content-security-policy-report-only"} +) + @router.get("/", include_in_schema=False) def read_root(): @@ -61,7 +65,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: request.url = request.url.replace('"', "").strip() if request.block_media: - async def block_media_route(route): + async def block_media_route(route) -> None: if route.request.resource_type in ("image", "media", "font"): await route.abort() else: @@ -69,6 +73,40 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await dep.page.route("**/*", block_media_route) + # The Firefox engine evaluates JS via eval(), which pages whose CSP lacks + # 'unsafe-eval' block - every page.evaluate() then fails with "call to + # eval() blocked by CSP". Rewrite document responses without CSP headers + # so the user-agent read and captcha solver work on CSP-locked pages. + # Juggler only routes the first request of a redirect chain, so follow + # redirects inside the fetch and record the final URL ourselves. + final_url: str | None = None + + async def strip_csp_route(route) -> None: + nonlocal final_url + if route.request.resource_type != "document": + await route.continue_() + return + try: + response = await route.fetch() + except Exception: + await route.continue_() + return + if route.request.frame == dep.page.main_frame: + final_url = response.url + if response.headers.get("content-security-policy") or response.headers.get( + "content-security-policy-report-only" + ): + headers = { + key: value + for key, value in response.headers.items() + if key.lower() not in CSP_HEADERS + } + await route.fulfill(response=response, headers=headers) + else: + await route.fulfill(response=response) + + await dep.page.route("**/*", strip_csp_route) + try: page_request = await dep.page.goto( request.url, timeout=timer.remaining() * 1000 @@ -130,7 +168,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: message="Success", solution=Solution( user_agent=await dep.page.evaluate("navigator.userAgent"), - url=dep.page.url, + url=final_url if final_url is not None else dep.page.url, status=status, cookies=cookies, headers=page_request.headers if page_request else {}, From e2fd2e6d42835dff4ec8dfb39ee0ccf43c1da173 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sat, 8 Aug 2026 00:31:25 +0200 Subject: [PATCH 2/2] refactor: simplify CSP stripping handler Drop error handling and conditional branches that duplicated the pass-through path; rely on the goto timeout as before. --- src/endpoints.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/src/endpoints.py b/src/endpoints.py index 5095874..5470e9e 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -73,12 +73,6 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await dep.page.route("**/*", block_media_route) - # The Firefox engine evaluates JS via eval(), which pages whose CSP lacks - # 'unsafe-eval' block - every page.evaluate() then fails with "call to - # eval() blocked by CSP". Rewrite document responses without CSP headers - # so the user-agent read and captcha solver work on CSP-locked pages. - # Juggler only routes the first request of a redirect chain, so follow - # redirects inside the fetch and record the final URL ourselves. final_url: str | None = None async def strip_csp_route(route) -> None: @@ -86,24 +80,17 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: if route.request.resource_type != "document": await route.continue_() return - try: - response = await route.fetch() - except Exception: - await route.continue_() - return + response = await route.fetch() if route.request.frame == dep.page.main_frame: final_url = response.url - if response.headers.get("content-security-policy") or response.headers.get( - "content-security-policy-report-only" - ): - headers = { + await route.fulfill( + response=response, + headers={ key: value for key, value in response.headers.items() if key.lower() not in CSP_HEADERS - } - await route.fulfill(response=response, headers=headers) - else: - await route.fulfill(response=response) + }, + ) await dep.page.route("**/*", strip_csp_route)