Merge pull request #395 from ThePhaseless/fix/csp-json-viewer-eval

fix: /v1 must survive CSP-blocked evaluate (incl. Firefox JSON viewer)

.github/workflows/docker-publish.yml

@@ -9,7 +9,7 @@ on:
   schedule:
     - cron: "25 0 * * *"
   push:
-    branches: ["*"]
+    branches: ["main"]
     # Publish semver tags as releases.
     tags: ["v*.*.*"]
     paths:
@@ -65,10 +65,12 @@ jobs:
         with:
           context: .
           platforms: linux/amd64
-          cache-from: type=gha,scope=x64
+          cache-from: type=gha,scope=amd64
           pull: true
-          cache-to: type=gha,mode=max,scope=x64
+          cache-to: type=gha,mode=max,scope=amd64
           target: test
+          build-args: |
+            GITHUB_BUILD=true
 
   build:
     needs: test
@@ -135,8 +137,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ matrix.platform }}
-          cache-from: type=gha,scope=${{ matrix.platform }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+          cache-from: type=gha,scope=${{ steps.vars.outputs.SURFIX }}
+          cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.SURFIX }}
           build-args: |
             GITHUB_BUILD=true
             VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.sha }}

Dockerfile

@@ -3,19 +3,16 @@
 # cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch).
 FROM ubuntu:24.04 AS base
 
-ARG GITHUB_BUILD=false \
-    VERSION
+ARG GITHUB_BUILD=false
 
 ENV GITHUB_BUILD=${GITHUB_BUILD}\
-    VERSION=${VERSION}\
-    DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
     # prevents python creating .pyc files
     PYTHONDONTWRITEBYTECODE=1 \
     UV_LINK_MODE=copy \
     PORT=8191 \
     XDG_CACHE_HOME=/cache \
-    HOME=/tmp
+    HOME=/home/byparr
 
 RUN apt-get update &&\
     apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
@@ -47,9 +44,9 @@ RUN mkdir -p /cache &&\
 
 COPY . .
 
-# Make app and cache world-readable; cache must be writable for runtime browser/profile data
-RUN chmod -R o+rX /app /cache &&\
-    chmod -R o+w /cache
+RUN mkdir -p /home/byparr &&\
+    chmod -R o+rX /app &&\
+    chmod -R a+rwX /cache /home/byparr
 
 FROM app AS test
 RUN \
@@ -57,6 +54,8 @@ RUN \
     uv run pytest --retries 3
 
 FROM app
+ARG VERSION
+ENV VERSION=${VERSION}
 USER 1000
 EXPOSE $PORT
 HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health"

README.md

@@ -17,6 +17,13 @@
 | `PROXY_USERNAME`     | None      | Username for proxy authentication.                                                                                                                                |
 | `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
 | `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |
+| `BROWSER_LOCALE`     | None      | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. |
+
+#### Browser language
+
+Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag like `en-US`, `de-DE`, `fr-FR`, `pl-PL`, or `zh-CN` to fix the browser's language and `Accept-Language` header. When unset, Byparr derives the locale from the egress country (e.g. a French proxy → `fr-FR`), keeping the browser language consistent with the exit IP.
+
+Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`.
 
 ## Proxy Recommendation
 

pyproject.toml

@@ -54,7 +54,8 @@ ignore = [
     "G004",
     "ANN001",
     "ANN204",
-    "ANN206",
+    "CPY001",
+    "BLE001",
 ]
 select = ["ALL"]
 extend-safe-fixes = ["D415"]

src/consts.py

@@ -3,8 +3,6 @@ import sys
 
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-from playwright_captcha import CaptchaType
-
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -24,6 +22,7 @@ class Settings(BaseSettings):
     block_media: bool = False
     return_only_cookies: bool = False
     owui_api_key: str | None = None
+    browser_locale: str | None = None
 
 
 settings = Settings()
@@ -44,12 +43,4 @@ BLOCK_MEDIA = settings.block_media
 RETURN_ONLY_COOKIES = settings.return_only_cookies
 
 OWUI_API_KEY = settings.owui_api_key
-
-CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = {
-    # Cloudflare
-    CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."],
-}
-
-CHALLENGE_TITLES = [
-    title for titles in CHALLENGE_TITLES_MAP.values() for title in titles
-]
+BROWSER_LOCALE = settings.browser_locale

src/endpoints.py

@@ -9,8 +9,10 @@ from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright_captcha import CaptchaType
+from playwright_captcha.solvers.click.cloudflare.utils.detection import (
+    detect_cloudflare_challenge,
+)
 
-from src.consts import CHALLENGE_TITLES
 from src.models import (
     HealthcheckResponse,
     LinkRequest,
@@ -67,12 +69,63 @@ async def health_check(sb: BrowserDep):
 async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     """Handle POST requests."""
     start_time = int(time.time() * 1000)
-
     timer = TimeoutTimer(duration=request.max_timeout)
-
     request.url = request.url.replace('"', "").strip()
 
+    final_url = await setup_routes(request, dep)
+
+    try:
+        challenge_detected, page_html, page_request, status = (
+            await _navigate_and_solve(dep, request, timer)
+        )
+    except (TimeoutError, PlaywrightTimeoutError) as e:
+        logger.error("Timed out while loading the page or solving the challenge")
+        raise HTTPException(
+            status_code=408,
+            detail="Timed out while loading the page or solving the challenge",
+        ) from e
+
+    cookies = await dep.context.cookies()
+    content_type, response_content = await build_response_content(
+        dep, request, page_request,
+        challenge_detected=challenge_detected,
+        page_html=page_html,
+    )
+
+    user_agent = (
+        page_request.request.headers.get("user-agent") if page_request else None
+    )
+    if user_agent is None:
+        try:
+            user_agent = await dep.page.evaluate("navigator.userAgent")
+        except Exception:
+            logger.warning("Could not determine User-Agent via page.evaluate")
+            user_agent = ""
+
+    return LinkResponse(
+        message="Success",
+        solution=Solution(
+            user_agent=user_agent,
+            url=final_url if final_url is not None else dep.page.url,
+            status=status,
+            cookies=cookies,
+            headers=page_request.headers if page_request else {},
+            response=response_content,
+            content_type=content_type,
+        ),
+        start_timestamp=start_time,
+    )
+
+
+async def setup_routes(request: LinkRequest, dep: BrowserDep) -> str | None:
+    """
+    Install request routes for media blocking and CSP stripping.
+
+    Returns the final URL captured during navigation; callers read it after
+    the page settles.
+    """
     if request.block_media:
+
         async def block_media_route(route) -> None:
             if route.request.resource_type in ("image", "media", "font"):
                 await route.abort()
@@ -107,79 +160,99 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
         )
 
     await dep.page.route("**/*", strip_csp_route)
+    return final_url
 
+
+async def _navigate_and_solve(
+    dep: BrowserDep,
+    request: LinkRequest,
+    timer: TimeoutTimer,
+) -> tuple[bool, str | None, object, HTTPStatus]:
+    """Navigate to the URL, then solve a challenge or wait for network idle."""
+    page_html: str | None = None
+    page_request = await dep.page.goto(
+        request.url, timeout=timer.remaining() * 1000
+    )
+    status = page_request.status if page_request else HTTPStatus.OK
+    await dep.page.wait_for_load_state(
+        state="domcontentloaded", timeout=timer.remaining() * 1000
+    )
+
+    challenge_active = (
+        await detect_cloudflare_challenge(dep.page, "interstitial")
+        or await detect_cloudflare_challenge(dep.page, "turnstile")
+    )
+    if not challenge_active:
+        page_html = await dep.page.content()
+        await _wait_for_networkidle(dep, timer)
+        return False, page_html, page_request, status
+
+    await _solve_challenge(dep, timer)
+    status = HTTPStatus.OK
+    return True, page_html, page_request, status
+
+
+async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None:
+    """Attempt to solve a detected Cloudflare interstitial challenge."""
+    logger.info("Challenge detected, attempting to solve...")
+    await wait_for(
+        dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
+            captcha_container=dep.page,
+            captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
+            wait_checkbox_attempts=1,
+            wait_checkbox_delay=0.5,
+        ),
+        timeout=timer.remaining(),
+    )
+    logger.debug("Challenge solved successfully.")
+
+
+async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:
+    """Wait for network idle, tolerating post-DOM-load stalls."""
     try:
-        page_request = await dep.page.goto(
-            request.url, timeout=timer.remaining() * 1000
-        )
-        status = page_request.status if page_request else HTTPStatus.OK
         await dep.page.wait_for_load_state(
-            state="domcontentloaded", timeout=timer.remaining() * 1000
+            "networkidle", timeout=timer.remaining() * 1000
+        )
+    except PlaywrightTimeoutError:
+        logger.info(
+            "networkidle timed out after domcontentloaded; "
+            "continuing with loaded page"
         )
 
-        if await dep.page.title() in CHALLENGE_TITLES:
-            logger.info("Challenge detected, attempting to solve...")
-            # Solve the captcha
-            await wait_for(
-                dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
-                    captcha_container=dep.page,
-                    captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
-                    wait_checkbox_attempts=1,
-                    wait_checkbox_delay=0.5,
-                ),
-                timeout=timer.remaining(),
-            )
-            status = HTTPStatus.OK
-            logger.debug("Challenge solved successfully.")
-        else:
-            try:
-                await dep.page.wait_for_load_state(
-                    "networkidle", timeout=timer.remaining() * 1000
-                )
-            except PlaywrightTimeoutError:
-                logger.info(
-                    "networkidle timed out after domcontentloaded; continuing with loaded page"
-                )
-    except (TimeoutError, PlaywrightTimeoutError) as e:
-        logger.error("Timed out while loading the page or solving the challenge")
-        raise HTTPException(
-            status_code=408,
-            detail="Timed out while loading the page or solving the challenge",
-        ) from e
-
-    cookies = await dep.context.cookies()
-
-    content_type = "text/html"
-    response_content = ""
 
+async def build_response_content(
+    dep: BrowserDep,
+    request: LinkRequest,
+    page_request: object,
+    *,
+    challenge_detected: bool,
+    page_html: str | None,
+) -> tuple[str, str]:
+    """Build (content_type, response_content) from the settled page."""
     if request.return_only_cookies:
-        response_content = ""
-    elif page_request and page_request.headers.get("content-type", "").startswith(
+        return "text/html", ""
+
+    if page_request and page_request.headers.get("content-type", "").startswith(
         "application/pdf"
     ):
-        content_type = "application/pdf"
-        try:
-            fetch_response = await dep.page.request.fetch(dep.page.url)
-            response_content = base64.b64encode(
-                await fetch_response.body()
-            ).decode("ascii")
-        except Exception:
-            logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
-            content_type = "text/html"
-            response_content = await dep.page.content()
-    else:
-        response_content = await dep.page.content()
+        return await _fetch_pdf_content(dep)
 
-    return LinkResponse(
-        message="Success",
-        solution=Solution(
-            user_agent=await dep.page.evaluate("navigator.userAgent"),
-            url=final_url if final_url is not None else dep.page.url,
-            status=status,
-            cookies=cookies,
-            headers=page_request.headers if page_request else {},
-            response=response_content,
-            content_type=content_type,
-        ),
-        start_timestamp=start_time,
+    response_content = (
+        page_html
+        if page_html is not None and not challenge_detected
+        else await dep.page.content()
     )
+    return "text/html", response_content
+
+
+async def _fetch_pdf_content(dep: BrowserDep) -> tuple[str, str]:
+    """Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
+    try:
+        fetch_response = await dep.page.request.fetch(dep.page.url)
+        response_content = base64.b64encode(
+            await fetch_response.body()
+        ).decode("ascii")
+    except Exception:
+        logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
+        return "text/html", await dep.page.content()
+    return "application/pdf", response_content

src/models.py

@@ -78,7 +78,7 @@ class LinkResponse(BaseModel):
     version: str = consts.VERSION
 
     @classmethod
-    def invalid(cls, url: str):
+    def invalid(cls, url: str) -> LinkResponse:
         """
         Return an invalid LinkResponse with default error values.
 

src/owui.py

@@ -69,7 +69,7 @@ async def load_urls(
             except PlaywrightTimeoutError:
                 logger.debug("networkidle timed out for %s; extracting anyway", url)
             content = await _extract_content(dep.page)
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             logger.warning("Failed to load %s: %s", url, exc)
             content = ""
         results.append(LoadResult(page_content=content, metadata={"source": url}))

src/utils.py

@@ -13,6 +13,7 @@ from playwright_captcha import (
 from pydantic import BaseModel, Field
 
 from src.consts import (
+    BROWSER_LOCALE,
     LOG_LEVEL,
     MAX_ATTEMPTS,
     PROXY_PASSWORD,
@@ -94,7 +95,8 @@ async def get_browser(
         headless=True,
         proxy=proxy_config,
         humanize=True,
-        locale="auto",
+        locale=BROWSER_LOCALE or "auto",
+        extra_prefs={"devtools.jsonview.enabled": False},
     ) as browser_raw:
         # InvisiblePlaywright yields a Browser instance
         browser = cast("Browser", browser_raw)

tests/main_test.py

@@ -1,3 +1,4 @@
+import base64
 from http import HTTPStatus
 from json import JSONDecodeError
 from unittest.mock import AsyncMock, MagicMock
@@ -58,6 +59,34 @@ def test_bypass(website: str):
     assert response.status_code == HTTPStatus.OK
 
 
+def test_json_api():
+    """JSON APIs must return 200, not crash on the UA evaluate.
+
+    Firefox renders application/json in a built-in viewer whose CSP blocks
+    Playwright's eval-based evaluate() (issue #394). The browser must be
+    launched with the viewer disabled so /v1 works and returns the raw JSON.
+    """
+    url = "https://api.ipify.org?format=json"
+    test_request = httpx2.get(url)
+    if test_request.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR:
+        pytest.skip(
+            f"Skipping JSON API test - upstream error ({test_request.status_code})"
+        )
+
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=url, cmd="request.get").model_dump(),
+    )
+
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping JSON API test - timed out (upstream issue)")
+
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    assert solution["userAgent"]
+    assert '"ip"' in solution["response"]
+
+
 def test_health_check():
     """
     Tests the health check endpoint.
@@ -83,7 +112,6 @@ def test_pdf_handling():
     if solution.get("contentType") != "application/pdf":
         pytest.skip("Skipping PDF test - PDF bytes could not be fetched (upstream issue)")
     assert solution["response"]  # non-empty base64
-    import base64
 
     decoded = base64.b64decode(solution["response"])
     assert decoded[:5] == b"%PDF-"
@@ -111,11 +139,16 @@ def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
     page = AsyncMock()
     page.url = "https://example.test/login"
     page.goto.return_value = MagicMock(
-        status=HTTPStatus.OK, headers={"content-type": "text/html"}
+        status=HTTPStatus.OK,
+        headers={"content-type": "text/html"},
+        request=MagicMock(headers={"user-agent": "UnitTestBrowser/1.0"}),
     )
     page.title.return_value = "Login"
     page.evaluate.return_value = "UnitTestBrowser/1.0"
     page.content.return_value = "<html><title>Login</title></html>"
+    locator = MagicMock()
+    locator.count = AsyncMock(return_value=0)
+    page.locator = MagicMock(return_value=locator)
 
     def wait_for_load_state(state: str, **_kwargs: object) -> None:
         """Fail the wait when asked for a configured state."""
@@ -133,14 +166,16 @@ def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
 @pytest.mark.asyncio
 async def test_networkidle_timeout_after_domcontentloaded_returns_content():
     """Pages that never go idle after DOM load must still return their content."""
+    dep = fake_dep(fail_states={"networkidle"})
     response = await read_item(
         LinkRequest(url="https://example.test/login"),
-        fake_dep(fail_states={"networkidle"}),
+        dep,
     )
 
     assert response.status == "ok"
     assert response.solution.status == HTTPStatus.OK
     assert response.solution.response == "<html><title>Login</title></html>"
+    dep.solver.solve_captcha.assert_not_called()
 
 
 @pytest.mark.asyncio
@@ -153,3 +188,22 @@ async def test_domcontentloaded_timeout_returns_408():
         )
 
     assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT
+
+
+@pytest.mark.asyncio
+async def test_user_agent_survives_csp_blocked_evaluate():
+    """UA comes from request headers when page CSP blocks evaluate (#394).
+
+    No CSP configuration (header, meta tag, or internal viewer document) may
+    turn /v1 into a 500.
+    """
+    dep = fake_dep()
+    dep.page.evaluate.side_effect = Exception("call to eval() blocked by CSP")
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        dep,
+    )
+
+    assert response.status == "ok"
+    assert response.solution.user_agent == "UnitTestBrowser/1.0"