Merge pull request #393 from ThePhaseless/chore/ruff-lint-cleanup

chore: fix ruff lint findings and refactor read_item

.devcontainer/devcontainer.json

@@ -21,7 +21,7 @@
   },
   "postCreateCommand": "uv sync --group test",
   "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+    "ghcr.io/devcontainers/features/docker-in-docker:4": {
       "moby": false
     }
   }

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -9,7 +9,6 @@ body:
       label: "I've completed the following steps:"
       options:
         - label: Read Loop Warning on Readme
-        - label: "Used --init/init: true in docker run/compose.yaml"
         - label: Done the troubleshooting from Readme
         - label: Checked if such issue already exists
         - label: Checked other websites with Cloudflare Turnstile

.github/workflows/docker-cleanup.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Delete PR Docker images
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const owner = context.repo.owner.toLowerCase();

.github/workflows/docker-publish.yml

@@ -9,7 +9,7 @@ on:
   schedule:
     - cron: "25 0 * * *"
   push:
-    branches: ["*"]
+    branches: ["main"]
     # Publish semver tags as releases.
     tags: ["v*.*.*"]
     paths:
@@ -54,7 +54,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
@@ -65,10 +65,12 @@ jobs:
         with:
           context: .
           platforms: linux/amd64
-          cache-from: type=gha,scope=x64
+          cache-from: type=gha,scope=amd64
           pull: true
-          cache-to: type=gha,mode=max,scope=x64
+          cache-to: type=gha,mode=max,scope=amd64
           target: test
+          build-args: |
+            GITHUB_BUILD=true
 
   build:
     needs: test
@@ -86,7 +88,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Prepare variables
         id: vars
@@ -135,8 +137,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ matrix.platform }}
-          cache-from: type=gha,scope=${{ matrix.platform }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+          cache-from: type=gha,scope=${{ steps.vars.outputs.SURFIX }}
+          cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.SURFIX }}
           build-args: |
             GITHUB_BUILD=true
             VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.sha }}
@@ -151,11 +153,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       # Install the cosign tool
       - name: Install cosign
-        uses: sigstore/cosign-installer@v3
+        uses: sigstore/cosign-installer@v4.1.2
 
       # Set up Docker Buildx
       - name: Set up Docker Buildx
@@ -184,6 +186,7 @@ jobs:
 
       # Create manifest lists and push
       - name: Create and push manifest lists
+        id: manifests
         run: |
           TAGS="${{ steps.meta.outputs.tags }}"
           args=""
@@ -213,11 +216,19 @@ jobs:
             ${image}:${{github.sha}}-arm64
           fi
 
-      # Sign the manifest
-      - name: Sign the manifests
+          # All tags created above alias a single manifest list; capture its digest
+          # so the signature is bound to the image bytes, not a mutable tag.
+          # Tags from metadata-action are full references (image:tag), one per line,
+          # so take the first line rather than splitting on spaces.
+          FIRST_TAG=$(printf '%s' "$TAGS" | head -n1)
+          DIGEST=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$FIRST_TAG")
+          echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
+
+      # Sign the manifest list by digest — every consumer tag aliases this digest
+      - name: Sign the manifest list by digest
         env:
-          TAGS: ${{ steps.meta.outputs.tags }}
+          IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          DIGEST: ${{ steps.manifests.outputs.DIGEST }}
         run: |
-          for TAG in $TAGS; do
-            cosign sign --yes $TAG
-          done
+          image=${IMAGE,,}
+          cosign sign --yes ${image}@${DIGEST}

Dockerfile

@@ -1,58 +1,62 @@
-# Ubuntu is required by playwright
-FROM ubuntu:latest AS base
+# Ubuntu is required by playwright.
+# Pin to 24.04 LTS: ubuntu:latest floats to 26.04, which Playwright 1.58
+# cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch).
+FROM ubuntu:24.04 AS base
 
-ARG GITHUB_BUILD=false \
-    UV_CACHE_DIR=/var/cache/uv \
-    VERSION \
-    USER=ubuntu \
-    UID=1000
-
-ARG GROUP=${USER} \
-    GID=${UID}
+ARG GITHUB_BUILD=false
 
 ENV GITHUB_BUILD=${GITHUB_BUILD}\
-    VERSION=${VERSION}\
-    DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
     # prevents python creating .pyc files
     PYTHONDONTWRITEBYTECODE=1 \
     UV_LINK_MODE=copy \
-    UV_CACHE_DIR=${UV_CACHE_DIR} \
-    PORT=8191
+    PORT=8191 \
+    XDG_CACHE_HOME=/cache \
+    HOME=/home/byparr
 
-RUN apt update &&\
-    apt -y upgrade &&\
-    apt install -y curl
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 FROM base AS devcontainer
-RUN apt install -y git &&\
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends git &&\
     uvx playwright install-deps firefox &&\
-    uvx camoufox fetch
+    uvx --from git+https://github.com/feder-cr/invisible_playwright.git python -m invisible_playwright fetch &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 ENTRYPOINT [ "sleep", "infinity" ]
 
 FROM base AS app
 WORKDIR /app
-RUN chown ${USER}:${GROUP} /app &&\
-    mkdir -p ${UV_CACHE_DIR} &&\
-    chown ${USER}:${GROUP} ${UV_CACHE_DIR}
 
-USER ${USER}
 COPY pyproject.toml uv.lock ./
-RUN uv sync && uv run camoufox fetch
-
-USER root
-RUN uv run playwright install-deps firefox
-USER ${USER}
+RUN mkdir -p /cache &&\
+    uv sync &&\
+    uv run python -m invisible_playwright fetch &&\
+    apt-get update &&\
+    uv run playwright install-deps firefox &&\
+    uv cache clean &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
+RUN mkdir -p /home/byparr &&\
+    chmod -R o+rX /app &&\
+    chmod -R a+rwX /cache /home/byparr
+
 FROM app AS test
 RUN \
     uv sync --group test &&\
     uv run pytest --retries 3
 
 FROM app
+ARG VERSION
+ENV VERSION=${VERSION}
+USER 1000
 EXPOSE $PORT
-HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://localhost:${PORT}/health"
-ENTRYPOINT ["uv", "run", "main.py"]
+HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health"
+ENTRYPOINT ["tini", "--", "/app/.venv/bin/python", "main.py"]

README.md

@@ -16,6 +16,14 @@
 | `PROXY_SERVER`       | None      | Proxy to use in format: `protocol://host:port`.                                                                                                                   |
 | `PROXY_USERNAME`     | None      | Username for proxy authentication.                                                                                                                                |
 | `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
+| `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |
+| `BROWSER_LOCALE`     | None      | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. |
+
+#### Browser language
+
+Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag like `en-US`, `de-DE`, `fr-FR`, `pl-PL`, or `zh-CN` to fix the browser's language and `Accept-Language` header. When unset, Byparr derives the locale from the egress country (e.g. a French proxy → `fr-FR`), keeping the browser language consistent with the exit IP.
+
+Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`.
 
 ## Proxy Recommendation
 
@@ -65,6 +73,20 @@ Once running, open:
 - `http://localhost:8191/docs`
 - `http://localhost:8191/` (redirects to `/docs`)
 
+### Open WebUI Integration
+
+Byparr can serve as an external web loader for [Open WebUI](https://github.com/open-webui/open-webui), allowing it to fetch web content through Byparr's anti-bot bypassing capabilities.
+
+Configure Open WebUI with these environment variables:
+
+```bash
+WEB_LOADER_ENGINE=external
+EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load
+EXTERNAL_WEB_LOADER_API_KEY=your-secret-key  # Optional, must match OWUI_API_KEY
+```
+
+The `/load` endpoint accepts `POST` requests with `{"urls": ["https://..."]}` and returns extracted text content for RAG pipelines.
+
 ## Troubleshooting
 
 ### Docker troubleshooting
@@ -75,6 +97,20 @@ Once running, open:
    1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
    2. If build fails, try troubleshooting on another host/using other method
 
+#### Proxmox OCI / LXC browser launch errors
+
+If you are running Byparr as an OCI container in Proxmox (or another LXC-based setup) and see a `FileNotFoundError` from `multiprocessing.synchronize`/`camoufox` when processing requests, increase the service's shared memory in `compose.yaml`:
+
+```yaml
+services:
+  byparr:
+    shm_size: 512mb
+    stdin_open: true
+    tty: true
+```
+
+`shm_size: 512mb` is usually enough; `stdin_open` and `tty` are only needed if your orchestrator runs the container without a TTY.
+
 ### Local troubleshooting
 
 1. Download [uv](https://docs.astral.sh/uv/getting-started/installation/)

compose.yaml

@@ -2,7 +2,6 @@ services:
   byparr:
     image: ghcr.io/thephaseless/byparr:latest
     restart: unless-stopped
-    init: true
     build:
       context: .
       dockerfile: Dockerfile

main.py

@@ -11,7 +11,8 @@ from fastapi.middleware.gzip import GZipMiddleware
 from src.consts import HOST, LOG_LEVEL, PORT, VERSION
 from src.endpoints import health_check, router
 from src.middlewares import LogRequest
-from src.utils import get_camoufox, logger
+from src.owui import router as owui_router
+from src.utils import get_browser, logger
 
 logger.info("Using version %s", VERSION)
 logger.info("Log level set to %s", logging.getLevelName(LOG_LEVEL))
@@ -21,11 +22,12 @@ app.add_middleware(GZipMiddleware)
 app.add_middleware(LogRequest)
 
 app.include_router(router=router)
+app.include_router(router=owui_router)
 
 
 async def init():
     """Initialize the application."""
-    async for browser in get_camoufox():
+    async for browser in get_browser():
         await health_check(browser)
 
 

pyproject.toml

@@ -8,27 +8,28 @@ version = "0.1.0"
 description = "API for getting cookies for Cloudflare challenges"
 readme = "README.md"
 dependencies = [
-    "camoufox[geoip]==0.4.*",
-    "fastapi[standard]==0.135.*",
+    "fastapi[standard]==0.141.*",
+    "invisible-playwright>=0.6.1",
+    "playwright==1.60.*",
     "playwright-captcha==0.1.*",
     "pydantic==2.*",
+    "pydantic-settings==2.*",
+    "trafilatura==2.2.*",
 ]
 urls = { repository = "https://github.com/ThePhaseless/Byparr" }
-
 [dependency-groups]
+
 test = [
-    "httpx==0.28.*",
-    "pytest==9.0.*",
-    "pytest-asyncio==1.3.*",
+    "httpx2==2.10.*",
+    "pytest==9.1.*",
+    "pytest-asyncio==1.4.*",
     "pytest-retry==1.7.*",
     "pytest-xdist==3.8.*",
     "setuptools>=80.9.0",
 ]
-dev = ["deptry==0.24.*", "ruff==0.15.*"]
-
+dev = ["deptry==0.25.*", "ruff==0.16.*"]
 [tool.deptry.per_rule_ignores]
 DEP002 = ["pyautogui"]
-
 [tool.ruff.lint]
 ignore = [
     "D203",
@@ -53,7 +54,8 @@ ignore = [
     "G004",
     "ANN001",
     "ANN204",
-    "ANN206",
+    "CPY001",
+    "BLE001",
 ]
 select = ["ALL"]
 extend-safe-fixes = ["D415"]

renovate.json

@@ -2,6 +2,16 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:recommended"],
   "forkProcessing": "enabled",
+  "lockFileMaintenance": {
+    "enabled": true,
+    "rebaseWhen": "behind-base-branch",
+    "branchTopic": "lock-file-maintenance",
+    "commitMessageAction": "Lock file maintenance",
+    "schedule": ["before 4am on monday"],
+    "prBodyDefinitions": {
+      "Change": "All locks refreshed"
+    }
+  },
   "packageRules": [
     {
       "automerge": true,

src/consts.py

@@ -1,32 +1,46 @@
 import logging
-import os
 import sys
-from pathlib import Path
 
-from playwright_captcha import CaptchaType
-from playwright_captcha.utils.camoufox_add_init_script.add_init_script import (
-    get_addon_path,
-)
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
-LOG_LEVEL = logging.getLevelNamesMapping()[os.getenv("LOG_LEVEL", "INFO").upper()]
 
-VERSION = os.getenv("VERSION", "unknown").removeprefix("v")
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
-ADDON_PATH = str(Path(get_addon_path()).absolute())
-MAX_ATTEMPTS = sys.maxsize
+    log_level: str = "INFO"
+    version: str = "unknown"
 
-PROXY_SERVER = os.getenv("PROXY_SERVER")
-PROXY_USERNAME = os.getenv("PROXY_USERNAME")
-PROXY_PASSWORD = os.getenv("PROXY_PASSWORD")
+    max_attempts: int = sys.maxsize
 
-HOST = os.getenv("HOST", "0.0.0.0")  # noqa: S104
-PORT = int(os.getenv("PORT", "8191"))
+    proxy_server: str | None = None
+    proxy_username: str | None = None
+    proxy_password: str | None = None
 
-CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = {
-    # Cloudflare
-    CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."],
-}
+    host: str = "0.0.0.0"  # noqa: S104
+    port: int = 8191
 
-CHALLENGE_TITLES = [
-    title for titles in CHALLENGE_TITLES_MAP.values() for title in titles
-]
+    block_media: bool = False
+    return_only_cookies: bool = False
+    owui_api_key: str | None = None
+    browser_locale: str | None = None
+
+
+settings = Settings()
+
+LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()]
+VERSION = settings.version.removeprefix("v")
+
+MAX_ATTEMPTS = settings.max_attempts
+
+PROXY_SERVER = settings.proxy_server
+PROXY_USERNAME = settings.proxy_username
+PROXY_PASSWORD = settings.proxy_password
+
+HOST = settings.host
+PORT = settings.port
+
+BLOCK_MEDIA = settings.block_media
+RETURN_ONLY_COOKIES = settings.return_only_cookies
+
+OWUI_API_KEY = settings.owui_api_key
+BROWSER_LOCALE = settings.browser_locale

src/endpoints.py

@@ -1,3 +1,4 @@
+import base64
 import time
 import warnings
 from asyncio import wait_for
@@ -6,23 +7,38 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright_captcha import CaptchaType
+from playwright_captcha.solvers.click.cloudflare.utils.detection import (
+    detect_cloudflare_challenge,
+)
 
-from src.consts import CHALLENGE_TITLES
 from src.models import (
     HealthcheckResponse,
     LinkRequest,
     LinkResponse,
     Solution,
 )
-from src.utils import CamoufoxDepClass, TimeoutTimer, get_camoufox, logger
+from src.utils import BrowserDepClass, TimeoutTimer, get_browser, logger
 
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
 
 router = APIRouter()
 
-CamoufoxDep = Annotated[CamoufoxDepClass, Depends(get_camoufox)]
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+# Headers to strip from the fulfilled response: CSP is removed for navigation
+# freedom, and content-encoding/content-length are stale once we request an
+# uncompressed body via accept-encoding: identity below.
+DROP_HEADERS = frozenset(
+    {
+        "content-security-policy",
+        "content-security-policy-report-only",
+        "content-encoding",
+        "content-length",
+    }
+)
 
 
 @router.get("/", include_in_schema=False)
@@ -33,7 +49,7 @@ def read_root():
 
 
 @router.get("/health")
-async def health_check(sb: CamoufoxDep):
+async def health_check(sb: BrowserDep):
     """Health check endpoint."""
     health_check_request = await read_item(
         LinkRequest.model_construct(url="https://google.com"),
@@ -50,57 +66,183 @@ async def health_check(sb: CamoufoxDep):
 
 
 @router.post("/v1")
-async def read_item(request: LinkRequest, dep: CamoufoxDep) -> LinkResponse:
+async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     """Handle POST requests."""
     start_time = int(time.time() * 1000)
-
     timer = TimeoutTimer(duration=request.max_timeout)
-
     request.url = request.url.replace('"', "").strip()
-    try:
-        page_request = await dep.page.goto(
-            request.url, timeout=timer.remaining() * 1000
-        )
-        status = page_request.status if page_request else HTTPStatus.OK
-        await dep.page.wait_for_load_state(
-            state="domcontentloaded", timeout=timer.remaining() * 1000
-        )
-        await dep.page.wait_for_load_state(
-            "networkidle", timeout=timer.remaining() * 1000
-        )
 
-        if await dep.page.title() in CHALLENGE_TITLES:
-            logger.info("Challenge detected, attempting to solve...")
-            # Solve the captcha
-            await wait_for(
-                dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
-                    captcha_container=dep.page,
-                    captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
-                    wait_checkbox_attempts=1,
-                    wait_checkbox_delay=0.5,
-                ),
-                timeout=timer.remaining(),
-            )
-            status = HTTPStatus.OK
-            logger.debug("Challenge solved successfully.")
-    except TimeoutError as e:
-        logger.error("Timed out while solving the challenge")
+    final_url = await setup_routes(request, dep)
+
+    try:
+        challenge_detected, page_html, page_request, status = (
+            await _navigate_and_solve(dep, request, timer)
+        )
+    except (TimeoutError, PlaywrightTimeoutError) as e:
+        logger.error("Timed out while loading the page or solving the challenge")
         raise HTTPException(
             status_code=408,
-            detail="Timed out while solving the challenge",
+            detail="Timed out while loading the page or solving the challenge",
         ) from e
 
     cookies = await dep.context.cookies()
+    content_type, response_content = await build_response_content(
+        dep, request, page_request,
+        challenge_detected=challenge_detected,
+        page_html=page_html,
+    )
 
     return LinkResponse(
         message="Success",
         solution=Solution(
             user_agent=await dep.page.evaluate("navigator.userAgent"),
-            url=dep.page.url,
+            url=final_url if final_url is not None else dep.page.url,
             status=status,
             cookies=cookies,
             headers=page_request.headers if page_request else {},
-            response=await dep.page.content(),
+            response=response_content,
+            content_type=content_type,
         ),
         start_timestamp=start_time,
     )
+
+
+async def setup_routes(request: LinkRequest, dep: BrowserDep) -> str | None:
+    """
+    Install request routes for media blocking and CSP stripping.
+
+    Returns the final URL captured during navigation; callers read it after
+    the page settles.
+    """
+    if request.block_media:
+
+        async def block_media_route(route) -> None:
+            if route.request.resource_type in ("image", "media", "font"):
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await dep.page.route("**/*", block_media_route)
+
+    final_url: str | None = None
+
+    async def strip_csp_route(route) -> None:
+        nonlocal final_url
+        if route.request.resource_type != "document":
+            await route.continue_()
+            return
+        # Request an uncompressed body via accept-encoding: identity. When
+        # route.fulfill re-serves the fetched response (by uid), it forwards
+        # the original compressed bytes; stripping content-encoding below
+        # would leave the browser reading compressed bytes as plain text.
+        response = await route.fetch(
+            headers={**route.request.headers, "accept-encoding": "identity"}
+        )
+        if route.request.frame == dep.page.main_frame:
+            final_url = response.url
+        await route.fulfill(
+            response=response,
+            headers={
+                key: value
+                for key, value in response.headers.items()
+                if key.lower() not in DROP_HEADERS
+            },
+        )
+
+    await dep.page.route("**/*", strip_csp_route)
+    return final_url
+
+
+async def _navigate_and_solve(
+    dep: BrowserDep,
+    request: LinkRequest,
+    timer: TimeoutTimer,
+) -> tuple[bool, str | None, object, HTTPStatus]:
+    """Navigate to the URL, then solve a challenge or wait for network idle."""
+    page_html: str | None = None
+    page_request = await dep.page.goto(
+        request.url, timeout=timer.remaining() * 1000
+    )
+    status = page_request.status if page_request else HTTPStatus.OK
+    await dep.page.wait_for_load_state(
+        state="domcontentloaded", timeout=timer.remaining() * 1000
+    )
+
+    challenge_active = (
+        await detect_cloudflare_challenge(dep.page, "interstitial")
+        or await detect_cloudflare_challenge(dep.page, "turnstile")
+    )
+    if not challenge_active:
+        page_html = await dep.page.content()
+        await _wait_for_networkidle(dep, timer)
+        return False, page_html, page_request, status
+
+    await _solve_challenge(dep, timer)
+    status = HTTPStatus.OK
+    return True, page_html, page_request, status
+
+
+async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None:
+    """Attempt to solve a detected Cloudflare interstitial challenge."""
+    logger.info("Challenge detected, attempting to solve...")
+    await wait_for(
+        dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
+            captcha_container=dep.page,
+            captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
+            wait_checkbox_attempts=1,
+            wait_checkbox_delay=0.5,
+        ),
+        timeout=timer.remaining(),
+    )
+    logger.debug("Challenge solved successfully.")
+
+
+async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:
+    """Wait for network idle, tolerating post-DOM-load stalls."""
+    try:
+        await dep.page.wait_for_load_state(
+            "networkidle", timeout=timer.remaining() * 1000
+        )
+    except PlaywrightTimeoutError:
+        logger.info(
+            "networkidle timed out after domcontentloaded; "
+            "continuing with loaded page"
+        )
+
+
+async def build_response_content(
+    dep: BrowserDep,
+    request: LinkRequest,
+    page_request: object,
+    *,
+    challenge_detected: bool,
+    page_html: str | None,
+) -> tuple[str, str]:
+    """Build (content_type, response_content) from the settled page."""
+    if request.return_only_cookies:
+        return "text/html", ""
+
+    if page_request and page_request.headers.get("content-type", "").startswith(
+        "application/pdf"
+    ):
+        return await _fetch_pdf_content(dep)
+
+    response_content = (
+        page_html
+        if page_html is not None and not challenge_detected
+        else await dep.page.content()
+    )
+    return "text/html", response_content
+
+
+async def _fetch_pdf_content(dep: BrowserDep) -> tuple[str, str]:
+    """Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
+    try:
+        fetch_response = await dep.page.request.fetch(dep.page.url)
+        response_content = base64.b64encode(
+            await fetch_response.body()
+        ).decode("ascii")
+    except Exception:
+        logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
+        return "text/html", await dep.page.content()
+    return "application/pdf", response_content

src/models.py

@@ -5,13 +5,17 @@ from http.client import INTERNAL_SERVER_ERROR
 from typing import Any
 
 from playwright.sync_api import Cookie
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic.alias_generators import to_camel
 
 from src import consts
 
+MS_PER_SECOND = 1000
+
 
 class LinkRequest(BaseModel):
+    model_config = {"populate_by_name": True}
+
     cmd: str = Field(
         default="request.get",
         description="Type of request, currently only supports GET requests. This string is purely for compatibility with FlareSolverr.",
@@ -19,8 +23,31 @@ class LinkRequest(BaseModel):
     url: str = Field(pattern=r"^https?://", default="https://")
     max_timeout: int = Field(
         default=60,
-        description="Maximum timeout in seconds for resolving the anti-bot challenge.",
+        alias="maxTimeout",
+        description=(
+            "Maximum timeout for resolving the anti-bot challenge. Values below 1000 "
+            "are treated as seconds; values of 1000 or more as milliseconds, matching "
+            "FlareSolverr's maxTimeout parameter."
+        ),
     )
+    block_media: bool = Field(
+        default=consts.BLOCK_MEDIA,
+        alias="blockMedia",
+        description="Block image, media, and font resources from loading.",
+    )
+    return_only_cookies: bool = Field(
+        default=consts.RETURN_ONLY_COOKIES,
+        alias="returnOnlyCookies",
+        description="Return only cookies, skip the page HTML content in the response.",
+    )
+
+    @field_validator("max_timeout")
+    @classmethod
+    def normalize_max_timeout(cls, value: int) -> int:
+        """Normalize FlareSolverr-style millisecond values to seconds."""
+        if value >= MS_PER_SECOND:
+            return value // MS_PER_SECOND
+        return value
 
 
 class HealthcheckResponse(BaseModel):
@@ -38,6 +65,7 @@ class Solution(BaseModel):
     user_agent: str = ""
     headers: dict[str, Any] = {}
     response: str = ""
+    content_type: str = Field(default="text/html", alias="contentType")
 
 
 class LinkResponse(BaseModel):
@@ -50,7 +78,7 @@ class LinkResponse(BaseModel):
     version: str = consts.VERSION
 
     @classmethod
-    def invalid(cls, url: str):
+    def invalid(cls, url: str) -> LinkResponse:
         """
         Return an invalid LinkResponse with default error values.
 

src/owui.py

@@ -0,0 +1,76 @@
+"""Open WebUI external web loader endpoint: POST /load."""
+
+from __future__ import annotations
+
+from hmac import compare_digest
+from typing import Annotated
+
+import trafilatura
+from fastapi import APIRouter, Depends, Header, HTTPException
+from playwright.async_api import Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from pydantic import BaseModel
+
+from src.consts import OWUI_API_KEY
+from src.utils import BrowserDepClass, get_browser, logger
+
+router = APIRouter(tags=["Open WebUI"])
+
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+
+class LoadRequest(BaseModel):
+    urls: list[str]
+
+
+class LoadResult(BaseModel):
+    page_content: str
+    metadata: dict[str, str]
+
+
+def require_auth(authorization: Annotated[str | None, Header()] = None) -> None:
+    """Enforce a bearer token on /load when OWUI_API_KEY is set."""
+    if not OWUI_API_KEY:
+        return
+    if authorization is None or not compare_digest(
+        authorization.encode(), f"Bearer {OWUI_API_KEY}".encode()
+    ):
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+async def _extract_content(page: Page) -> str:
+    """Return the page's main article text, falling back to visible text."""
+    article = trafilatura.extract(await page.content())
+    if article:
+        return article
+    result = await page.evaluate("() => document.body ? document.body.innerText : ''")
+    return "\n".join(line.strip() for line in result.splitlines() if line.strip())
+
+
+@router.post("/load", response_model=list[LoadResult])
+async def load_urls(
+    request: LoadRequest,
+    _auth: Annotated[None, Depends(require_auth)],
+    dep: BrowserDep,
+) -> list[LoadResult]:
+    """
+    Fetch URLs through the anti-bot browser and return their text content.
+
+    Each URL is fetched sequentially; a failing URL yields empty
+    page_content so Open WebUI's RAG pipeline degrades gracefully.
+    """
+    results: list[LoadResult] = []
+    for url in request.urls:
+        try:
+            await dep.page.goto(url, timeout=60_000)
+            await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            try:
+                await dep.page.wait_for_load_state("networkidle", timeout=15_000)
+            except PlaywrightTimeoutError:
+                logger.debug("networkidle timed out for %s; extracting anyway", url)
+            content = await _extract_content(dep.page)
+        except Exception as exc:
+            logger.warning("Failed to load %s: %s", url, exc)
+            content = ""
+        results.append(LoadResult(page_content=content, metadata={"source": url}))
+    return results

src/utils.py

@@ -3,8 +3,8 @@ import time
 from collections.abc import AsyncGenerator
 from typing import Annotated, NamedTuple, cast
 
-from camoufox import AsyncCamoufox
 from fastapi import Header
+from invisible_playwright.async_api import InvisiblePlaywright
 from playwright.async_api import Browser, BrowserContext, Page
 from playwright_captcha import (
     ClickSolver,
@@ -13,7 +13,7 @@ from playwright_captcha import (
 from pydantic import BaseModel, Field
 
 from src.consts import (
-    ADDON_PATH,
+    BROWSER_LOCALE,
     LOG_LEVEL,
     MAX_ATTEMPTS,
     PROXY_PASSWORD,
@@ -44,13 +44,13 @@ class TimeoutTimer(BaseModel):
         return max(0, self.duration - (time.perf_counter() - self.start_time))
 
 
-class CamoufoxDepClass(NamedTuple):
+class BrowserDepClass(NamedTuple):
     page: Page
     solver: ClickSolver
     context: BrowserContext
 
 
-async def get_camoufox(
+async def get_browser(
     x_proxy_server: Annotated[
         str | None,
         Header(
@@ -70,8 +70,8 @@ async def get_camoufox(
             alias="X-Proxy-Password",
         ),
     ] = None,
-) -> AsyncGenerator[CamoufoxDepClass]:
-    """Get Camoufox instance."""
+) -> AsyncGenerator[BrowserDepClass]:
+    """Get InvisiblePlaywright browser instance."""
     header_server = x_proxy_server
     header_username = x_proxy_username
     header_password = x_proxy_password
@@ -91,26 +91,20 @@ async def get_camoufox(
             "password": PROXY_PASSWORD,
         }
 
-    async with AsyncCamoufox(
-        main_world_eval=True,
-        addons=[ADDON_PATH],
-        geoip=True,
-        proxy=proxy_config,
-        locale="en-US",
+    async with InvisiblePlaywright(
         headless=True,
+        proxy=proxy_config,
         humanize=True,
-        i_know_what_im_doing=True,
-        config={"forceScopeAccess": True},  # add this when creating Camoufox instance
-        disable_coop=True,  # add this when creating Camoufox instance
+        locale=BROWSER_LOCALE or "auto",
     ) as browser_raw:
-        # Cast to Browser since AsyncCamoufox always returns a Browser, not BrowserContext
+        # InvisiblePlaywright yields a Browser instance
         browser = cast("Browser", browser_raw)
         context = await browser.new_context()
         page = await context.new_page()
         async with ClickSolver(
-            framework=FrameworkType.CAMOUFOX,
+            framework=FrameworkType.PLAYWRIGHT,
             page=page,
             max_attempts=MAX_ATTEMPTS,
             attempt_delay=1,
         ) as solver:
-            yield CamoufoxDepClass(page, solver, context)
+            yield BrowserDepClass(page, solver, context)

tests/main_test.py

@@ -1,12 +1,18 @@
+import base64
 from http import HTTPStatus
 from json import JSONDecodeError
+from unittest.mock import AsyncMock, MagicMock
 
-import httpx
+import httpx2
 import pytest
+from fastapi import HTTPException
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
+from src.endpoints import read_item
 from src.models import LinkRequest
+from src.utils import BrowserDepClass
 
 client = TestClient(app)
 
@@ -27,7 +33,7 @@ def test_bypass(website: str):
 
     This test is skipped if the website is not reachable or does not have cloudflare/DDOS-GUARD.
     """
-    test_request = httpx.get(
+    test_request = httpx2.get(
         website,
     )
     if (
@@ -47,6 +53,9 @@ def test_bypass(website: str):
         json=LinkRequest.model_construct(url=website, cmd="request.get").model_dump(),
     )
 
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip(f"Skipping {website} - timed out (upstream issue)")
+
     assert response.status_code == HTTPStatus.OK
 
 
@@ -59,3 +68,93 @@ def test_health_check():
     """
     response = client.get("/health")
     assert response.status_code == HTTPStatus.OK
+
+
+def test_pdf_handling():
+    """Tests that PDF URLs return the raw PDF bytes, not the Firefox viewer HTML."""
+    pdf_url = "https://mondaymandala.com/wp-content/uploads/Mickey-And-Minnie-Mouse-Holding-An-Easter-Egg-Basket-Coloring-Page-For-Kids.pdf"
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=pdf_url, cmd="request.get").model_dump(),
+    )
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping PDF test - timed out (upstream issue)")
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    if solution.get("contentType") != "application/pdf":
+        pytest.skip("Skipping PDF test - PDF bytes could not be fetched (upstream issue)")
+    assert solution["response"]  # non-empty base64
+
+    decoded = base64.b64decode(solution["response"])
+    assert decoded[:5] == b"%PDF-"
+
+
+@pytest.mark.parametrize(
+    ("payload", "expected"),
+    [
+        ({"max_timeout": 60}, 60),  # native API: seconds
+        ({"maxTimeout": 60}, 60),  # FlareSolverr alias, seconds-range value
+        ({"maxTimeout": 60000}, 60),  # FlareSolverr alias: milliseconds
+        ({"maxTimeout": 55000}, 55),
+        ({"maxTimeout": 1000}, 1),
+        ({}, 60),  # default
+    ],
+)
+def test_max_timeout_normalization(payload: dict, expected: int):
+    """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds."""
+    request = LinkRequest(url="https://example.com", **payload)
+    assert request.max_timeout == expected
+
+
+def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
+    """Build a browser dependency triple backed by mocks."""
+    page = AsyncMock()
+    page.url = "https://example.test/login"
+    page.goto.return_value = MagicMock(
+        status=HTTPStatus.OK, headers={"content-type": "text/html"}
+    )
+    page.title.return_value = "Login"
+    page.evaluate.return_value = "UnitTestBrowser/1.0"
+    page.content.return_value = "<html><title>Login</title></html>"
+    locator = MagicMock()
+    locator.count = AsyncMock(return_value=0)
+    page.locator = MagicMock(return_value=locator)
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        """Fail the wait when asked for a configured state."""
+        if state in (fail_states or set()):
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+
+    context = AsyncMock()
+    context.cookies.return_value = []
+    return BrowserDepClass(page=page, solver=AsyncMock(), context=context)
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_after_domcontentloaded_returns_content():
+    """Pages that never go idle after DOM load must still return their content."""
+    dep = fake_dep(fail_states={"networkidle"})
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        dep,
+    )
+
+    assert response.status == "ok"
+    assert response.solution.status == HTTPStatus.OK
+    assert response.solution.response == "<html><title>Login</title></html>"
+    dep.solver.solve_captcha.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_domcontentloaded_timeout_returns_408():
+    """Fatal timeouts during initial page load still return a controlled 408."""
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login"),
+            fake_dep(fail_states={"domcontentloaded"}),
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT

tests/owui_test.py

@@ -0,0 +1,112 @@
+from http import HTTPStatus
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from starlette.testclient import TestClient
+
+from main import app
+from src.owui import LoadRequest, load_urls
+from src.utils import BrowserDepClass
+
+client = TestClient(app)
+
+
+def test_owui_load_basic():
+    """/load returns one result per URL with the expected shape."""
+    response = client.post("/load", json={"urls": ["https://example.com"]})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"]
+    assert results[0]["metadata"] == {"source": "https://example.com"}
+
+
+def test_owui_load_multiple_urls():
+    """/load returns one result per URL, in order."""
+    urls = ["https://example.com", "https://example.org"]
+    response = client.post("/load", json={"urls": urls})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert [r["metadata"]["source"] for r in results] == urls
+
+
+def test_owui_load_invalid_url_graceful():
+    """Unreachable URLs yield empty page_content instead of an error."""
+    response = client.post(
+        "/load", json={"urls": ["https://this-domain-does-not-exist-12345.invalid"]}
+    )
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"] == ""
+
+
+@pytest.mark.parametrize(
+    "headers",
+    [None, {"Authorization": "Bearer wrong-key"}],
+)
+def test_owui_load_rejects_missing_or_wrong_key(headers):
+    """/load returns 401 without a valid bearer token when a key is set."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load", json={"urls": ["https://example.com"]}, headers=headers
+        )
+        assert response.status_code == HTTPStatus.UNAUTHORIZED
+
+
+def test_owui_load_accepts_valid_key():
+    """/load succeeds with the configured bearer token."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load",
+            json={"urls": ["https://example.com"]},
+            headers={"Authorization": "Bearer test-secret-key"},
+        )
+        assert response.status_code == HTTPStatus.OK
+
+
+ARTICLE_HTML = """<html><head><title>Test</title></head><body>
+<article><h1>Example Title</h1><p>This is the main article body with enough words for
+trafilatura to consider it real content rather than boilerplate.</p></article>
+<nav><a href="/x">nav link</a></nav>
+</body></html>"""
+
+
+def fake_dep(*, html: str = ARTICLE_HTML) -> BrowserDepClass:
+    """Browser dependency whose page loads HTML but never reaches networkidle."""
+    page = AsyncMock()
+    page.goto.return_value = MagicMock()
+    page.content.return_value = html
+    page.evaluate.return_value = "line one\n\nline two"
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        if state == "networkidle":
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+    return BrowserDepClass(page=page, solver=AsyncMock(), context=AsyncMock())
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_still_extracts_content():
+    """A page that never reaches networkidle still yields its article text."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]), None, fake_dep()
+    )
+    assert results[0].page_content == (
+        "Example TitleThis is the main article body with enough words for "
+        "trafilatura to consider it real content rather than boilerplate."
+    )
+
+
+@pytest.mark.asyncio
+async def test_extraction_falls_back_to_innertext():
+    """Pages trafilatura cannot score fall back to the rendered innerText."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]),
+        None,
+        fake_dep(html="<html><body></body></html>"),
+    )
+    assert results[0].page_content == "line one\nline two"