Add sponsor badge to README

Added a sponsor badge to the README.

.devcontainer/devcontainer.json

@@ -21,7 +21,7 @@
   },
   "postCreateCommand": "uv sync --group test",
   "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+    "ghcr.io/devcontainers/features/docker-in-docker:4": {
       "moby": false
     }
   }

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# Supported funding model platforms
+
+github: [ThePhaseless]

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -9,7 +9,6 @@ body:
       label: "I've completed the following steps:"
       options:
         - label: Read Loop Warning on Readme
-        - label: "Used --init/init: true in docker run/compose.yaml"
         - label: Done the troubleshooting from Readme
         - label: Checked if such issue already exists
         - label: Checked other websites with Cloudflare Turnstile
@@ -27,8 +26,8 @@ body:
     attributes:
       label: "The issue is still present in the latest main tag:"
       options:
-        - label: Yes
+        - label: "Yes"
-        - label: No
+        - label: "No"
 
   - type: input
     id: docker-host

.github/workflows/docker-cleanup.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Delete PR Docker images
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const owner = context.repo.owner.toLowerCase();

.github/workflows/docker-publish.yml

@@ -9,7 +9,7 @@ on:
   schedule:
     - cron: "25 0 * * *"
   push:
-    branches: ["*"]
+    branches: ["main"]
     # Publish semver tags as releases.
     tags: ["v*.*.*"]
     paths:
@@ -54,21 +54,23 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Test
         id: test
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64
-          cache-from: type=gha,scope=x64
+          cache-from: type=gha,scope=amd64
           pull: true
-          cache-to: type=gha,mode=max,scope=x64
+          cache-to: type=gha,mode=max,scope=amd64
           target: test
+          build-args: |
+            GITHUB_BUILD=true
 
   build:
     needs: test
@@ -86,7 +88,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Prepare variables
         id: vars
@@ -102,15 +104,15 @@ jobs:
           fi
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       # Set up BuildKit Docker container builder
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Log into registry
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -119,7 +121,7 @@ jobs:
       # Extract metadata (tags, labels) for Docker
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           tags: type=raw,value=${{ steps.vars.outputs.LOCAL_TAG }}
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -127,7 +129,7 @@ jobs:
       # Build and push Docker image for each platform
       - name: Build Docker image
         id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           pull: true
@@ -135,8 +137,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ matrix.platform }}
-          cache-from: type=gha,scope=${{ matrix.platform }}
+          cache-from: type=gha,scope=${{ steps.vars.outputs.SURFIX }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.SURFIX }}
           build-args: |
             GITHUB_BUILD=true
             VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.sha }}
@@ -151,19 +153,19 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       # Install the cosign tool
       - name: Install cosign
-        uses: sigstore/cosign-installer@v3
+        uses: sigstore/cosign-installer@v4.1.2
 
       # Set up Docker Buildx
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Log into registry
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -172,7 +174,7 @@ jobs:
       # Extract Docker metadata for tagging
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           tags: |
             type=ref,event=branch
@@ -184,6 +186,7 @@ jobs:
 
       # Create manifest lists and push
       - name: Create and push manifest lists
+        id: manifests
         run: |
           TAGS="${{ steps.meta.outputs.tags }}"
           args=""
@@ -213,11 +216,19 @@ jobs:
             ${image}:${{github.sha}}-arm64
           fi
 
-      # Sign the manifest
+          # All tags created above alias a single manifest list; capture its digest
-      - name: Sign the manifests
+          # so the signature is bound to the image bytes, not a mutable tag.
+          # Tags from metadata-action are full references (image:tag), one per line,
+          # so take the first line rather than splitting on spaces.
+          FIRST_TAG=$(printf '%s' "$TAGS" | head -n1)
+          DIGEST=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$FIRST_TAG")
+          echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
+
+      # Sign the manifest list by digest — every consumer tag aliases this digest
+      - name: Sign the manifest list by digest
         env:
-          TAGS: ${{ steps.meta.outputs.tags }}
+          IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          DIGEST: ${{ steps.manifests.outputs.DIGEST }}
         run: |
-          for TAG in $TAGS; do
+          image=${IMAGE,,}
-            cosign sign --yes $TAG
+          cosign sign --yes ${image}@${DIGEST}
-          done

AGENTS.md

@@ -2,14 +2,14 @@
 
 ## Project overview
 
-- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using Camoufox + Playwright captcha solver.
+- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using invisible_playwright.
-- Entry point: main app in main.py; routes and request flow defined in src/endpoints.py and src/models.py.
+- Entry point: main app in main.py; routes and request flow in src/endpoints.py, challenge handling in src/challenge.py, response bodies in src/content.py, models in src/models.py.
-- Browser lifecycle is owned by get_camoufox() in src/utils.py, which yields a page, solver, and context for each request.
+- Browser lifecycle is owned by get_browser() in src/utils.py, which yields a page and context for each request.
 
 ## Architecture and data flow
 
-- Request flow: POST /v1 -> read_item() -> browser.goto() -> wait for load states -> detect challenge title -> solve captcha -> return LinkResponse.
+- Request flow: POST /v1 -> read_item() -> page.goto() -> wait for load states -> detect the interstitial -> click its checkbox until it clears -> return LinkResponse.
-- Challenge detection uses CHALLENGE_TITLES from src/consts.py; update this map when adding providers.
+- Challenge detection uses detect_cloudflare_challenge() from playwright_captcha; the challenge is over when its markup goes, not when a solver says so.
 - Health check hits /v1 internally with <https://google.com> and fails if status is not OK.
 - Logging: LogRequest middleware logs only POST /v1 timing and outcome; other paths pass through.
 
@@ -28,7 +28,7 @@
 
 - Local run: uv sync && uv run main.py
 - Init mode: uv run main.py --init (pre-warms health check via browser setup)
-- Tests: uv sync --group test && uv run pytest --retries 3
+- Tests: uv sync --group test && uv run pytest --retries 5
 - Docker troubleshooting: docker build --target test .
 
 ## Tests and external dependencies

Dockerfile

@@ -1,57 +1,62 @@
-# Ubuntu is required by playwright
+# Ubuntu is required by playwright.
-FROM ubuntu:latest AS base
+# Pin to 24.04 LTS: ubuntu:latest floats to 26.04, which Playwright 1.58
+# cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch).
+FROM ubuntu:24.04 AS base
 
-ARG GITHUB_BUILD=false \
+ARG GITHUB_BUILD=false
-    UV_CACHE_DIR=/var/cache/uv \
-    VERSION \
-    USER=ubuntu \
-    UID=1000
-
-ARG GROUP=${USER} \
-    GID=${UID}
 
 ENV GITHUB_BUILD=${GITHUB_BUILD}\
-    VERSION=${VERSION}\
-    DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
     # prevents python creating .pyc files
     PYTHONDONTWRITEBYTECODE=1 \
     UV_LINK_MODE=copy \
-    UV_CACHE_DIR=${UV_CACHE_DIR}
+    PORT=8191 \
+    XDG_CACHE_HOME=/cache \
+    HOME=/home/byparr
 
-RUN apt update &&\
+RUN apt-get update &&\
-    apt -y upgrade &&\
+    apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
-    apt install -y curl
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 FROM base AS devcontainer
-RUN apt install -y git &&\
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends git &&\
     uvx playwright install-deps firefox &&\
-    uvx camoufox fetch
+    uvx --from git+https://github.com/feder-cr/invisible_playwright.git python -m invisible_playwright fetch &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 ENTRYPOINT [ "sleep", "infinity" ]
 
 FROM base AS app
 WORKDIR /app
-RUN chown ${USER}:${GROUP} /app &&\
-    mkdir -p ${UV_CACHE_DIR} &&\
-    chown ${USER}:${GROUP} ${UV_CACHE_DIR}
 
-USER ${USER}
 COPY pyproject.toml uv.lock ./
-RUN uv sync && uv run camoufox fetch
+RUN mkdir -p /cache &&\
-
+    uv sync &&\
-USER root
+    uv run python -m invisible_playwright fetch &&\
-RUN uv run playwright install-deps firefox
+    apt-get update &&\
-USER ${USER}
+    uv run playwright install-deps firefox &&\
+    uv cache clean &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
+RUN mkdir -p /home/byparr &&\
+    chmod -R o+rX /app &&\
+    chmod -R a+rwX /cache /home/byparr
+
 FROM app AS test
 RUN \
     uv sync --group test &&\
-    uv run pytest --retries 3
+    uv run pytest -rs --retries 5
 
 FROM app
-EXPOSE 8191
+ARG VERSION
-HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
+ENV VERSION=${VERSION}
-ENTRYPOINT ["uv", "run", "main.py"]
+USER 1000
+EXPOSE $PORT
+HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health"
+ENTRYPOINT ["tini", "--", "/app/.venv/bin/python", "main.py"]

README.md

@@ -1,4 +1,4 @@
-# Byparr
+# Byparr [![Sponsor](https://img.shields.io/badge/Sponsor-%E2%9D%A4-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/ThePhaseless)
 
 <p align="center">
    <img src="icon/logo-byparr.svg" alt="Byparr logo" width="120" />
@@ -16,6 +16,14 @@
 | `PROXY_SERVER`       | None      | Proxy to use in format: `protocol://host:port`.                                                                                                                   |
 | `PROXY_USERNAME`     | None      | Username for proxy authentication.                                                                                                                                |
 | `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
+| `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |
+| `BROWSER_LOCALE`     | None      | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. |
+
+#### Browser language
+
+Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag like `en-US`, `de-DE`, `fr-FR`, `pl-PL`, or `zh-CN` to fix the browser's language and `Accept-Language` header. When unset, Byparr derives the locale from the egress country (e.g. a French proxy → `fr-FR`), keeping the browser language consistent with the exit IP.
+
+Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`.
 
 ## Proxy Recommendation
 
@@ -45,17 +53,18 @@ docker compose up -d
 
 1. Pull and run the image:
 
-```bash
+   ```bash
-docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
+   docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
-```
+   ```
 
-1. Optional: set env vars using `-e` or `--env-file`.
+2. Optional: set env vars using `-e` or `--env-file`.
 
 ### Local install
 
-1. Install [uv](https://docs.astral.sh/uv/getting-started/installation/).
+1. Install ([or update when Python version changes](https://github.com/astral-sh/uv/issues/17887)) [uv](https://docs.astral.sh/uv/getting-started/installation/).
-2. Run `uv run main.py`
+2. Clone this repo - `git clone https://github.com/ThePhaseless/Byparr`
-3. Profit.
+3. Run `uv run main.py`
+4. Enjoy!
 
 ### API Docs
 
@@ -64,6 +73,20 @@ Once running, open:
 - `http://localhost:8191/docs`
 - `http://localhost:8191/` (redirects to `/docs`)
 
+### Open WebUI Integration
+
+Byparr can serve as an external web loader for [Open WebUI](https://github.com/open-webui/open-webui), allowing it to fetch web content through Byparr's anti-bot bypassing capabilities.
+
+Configure Open WebUI with these environment variables:
+
+```bash
+WEB_LOADER_ENGINE=external
+EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load
+EXTERNAL_WEB_LOADER_API_KEY=your-secret-key  # Optional, must match OWUI_API_KEY
+```
+
+The `/load` endpoint accepts `POST` requests with `{"urls": ["https://..."]}` and returns extracted text content for RAG pipelines.
+
 ## Troubleshooting
 
 ### Docker troubleshooting
@@ -74,6 +97,20 @@ Once running, open:
    1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
    2. If build fails, try troubleshooting on another host/using other method
 
+#### Proxmox OCI / LXC browser launch errors
+
+If you are running Byparr as an OCI container in Proxmox (or another LXC-based setup) and see a `FileNotFoundError` from `multiprocessing.synchronize`/`camoufox` when processing requests, increase the service's shared memory in `compose.yaml`:
+
+```yaml
+services:
+  byparr:
+    shm_size: 512mb
+    stdin_open: true
+    tty: true
+```
+
+`shm_size: 512mb` is usually enough; `stdin_open` and `tty` are only needed if your orchestrator runs the container without a TTY.
+
 ### Local troubleshooting
 
 1. Download [uv](https://docs.astral.sh/uv/getting-started/installation/)

compose.yaml

@@ -2,7 +2,8 @@ services:
   byparr:
     image: ghcr.io/thephaseless/byparr:latest
     restart: unless-stopped
-    init: true
+    # environment:
+    #   LOG_LEVEL: debug
     build:
       context: .
       dockerfile: Dockerfile

main.py

@@ -11,7 +11,8 @@ from fastapi.middleware.gzip import GZipMiddleware
 from src.consts import HOST, LOG_LEVEL, PORT, VERSION
 from src.endpoints import health_check, router
 from src.middlewares import LogRequest
-from src.utils import get_camoufox, logger
+from src.owui import router as owui_router
+from src.utils import get_browser, logger
 
 logger.info("Using version %s", VERSION)
 logger.info("Log level set to %s", logging.getLevelName(LOG_LEVEL))
@@ -21,11 +22,12 @@ app.add_middleware(GZipMiddleware)
 app.add_middleware(LogRequest)
 
 app.include_router(router=router)
+app.include_router(router=owui_router)
 
 
 async def init():
     """Initialize the application."""
-    async for browser in get_camoufox():
+    async for browser in get_browser():
         await health_check(browser)
 
 

pyproject.toml

@@ -8,27 +8,28 @@ version = "0.1.0"
 description = "API for getting cookies for Cloudflare challenges"
 readme = "README.md"
 dependencies = [
-    "camoufox[geoip]==0.4.*",
+    "fastapi[standard]==0.141.*",
-    "fastapi[standard]==0.128.*",
+    "invisible-playwright>=0.6.1",
+    "playwright==1.63.*",
     "playwright-captcha==0.1.*",
     "pydantic==2.*",
+    "pydantic-settings==2.*",
+    "trafilatura==2.2.*",
 ]
 urls = { repository = "https://github.com/ThePhaseless/Byparr" }
-
 [dependency-groups]
+
 test = [
-    "httpx==0.28.*",
+    "httpx2==2.13.*",
-    "pytest==9.0.*",
+    "pytest==9.1.*",
-    "pytest-asyncio==1.3.*",
+    "pytest-asyncio==1.4.*",
     "pytest-retry==1.7.*",
     "pytest-xdist==3.8.*",
     "setuptools>=80.9.0",
 ]
-dev = ["deptry==0.24.*", "ruff==0.15.*"]
+dev = ["deptry==0.25.*", "ruff==0.16.*"]
-
 [tool.deptry.per_rule_ignores]
 DEP002 = ["pyautogui"]
-
 [tool.ruff.lint]
 ignore = [
     "D203",
@@ -53,7 +54,8 @@ ignore = [
     "G004",
     "ANN001",
     "ANN204",
-    "ANN206",
+    "CPY001",
+    "BLE001",
 ]
 select = ["ALL"]
 extend-safe-fixes = ["D415"]

renovate.json

@@ -2,6 +2,16 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:recommended"],
   "forkProcessing": "enabled",
+  "lockFileMaintenance": {
+    "enabled": true,
+    "rebaseWhen": "behind-base-branch",
+    "branchTopic": "lock-file-maintenance",
+    "commitMessageAction": "Lock file maintenance",
+    "schedule": ["before 4am on monday"],
+    "prBodyDefinitions": {
+      "Change": "All locks refreshed"
+    }
+  },
   "packageRules": [
     {
       "automerge": true,

src/challenge.py

@@ -0,0 +1,117 @@
+import time
+from asyncio import sleep
+from contextlib import suppress
+
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import FloatRect, Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from playwright_captcha.solvers.click.cloudflare.utils.detection import (
+    CF_INTERSTITIAL_INDICATORS_SELECTORS,
+    detect_cloudflare_challenge,
+)
+
+from src.utils import TimeoutTimer, logger
+
+__all__ = [
+    "CF_INTERSTITIAL_INDICATORS_SELECTORS",
+    "challenge_present",
+    "solve_challenge",
+]
+
+POLL_INTERVAL = 0.25
+CLICK_SETTLE = 1.5
+CLICK_COOLDOWN = 4
+PROBE_INTERVAL = 0.5
+TOKEN_READ_TIMEOUT = 1000
+BOX_READ_TIMEOUT = 1000
+CHECKBOX_INSET = 25
+TURNSTILE_INPUT = 'input[name="cf-turnstile-response"]'
+WIDGET_ANCESTOR_DEPTHS = (1, 2, 3, 4)
+WIDGET_MIN_WIDTH = 40
+WIDGET_MIN_HEIGHT = 20
+WIDGET_MAX_HEIGHT = 120
+
+
+async def challenge_present(page: Page) -> bool:
+    """Report whether the Cloudflare interstitial is up."""
+    return await detect_cloudflare_challenge(page, "interstitial")
+
+
+async def widget_box(page: Page) -> FloatRect | None:
+    """Measure the widget container with locators; running page scripts resets the challenge."""
+    for depth in WIDGET_ANCESTOR_DEPTHS:
+        widget = page.locator(f"{TURNSTILE_INPUT} >> xpath=ancestor::div[{depth}]")
+        with suppress(PlaywrightError, PlaywrightTimeoutError):
+            if await widget.count() == 0:
+                continue
+            box = await widget.first.bounding_box(timeout=BOX_READ_TIMEOUT)
+            if (
+                box
+                and box["width"] > WIDGET_MIN_WIDTH
+                and WIDGET_MIN_HEIGHT < box["height"] < WIDGET_MAX_HEIGHT
+            ):
+                return box
+    return None
+
+
+async def click_checkbox(page: Page) -> bool:
+    """Click the checkbox through its container, leaving its closed shadow root alone."""
+    box = await widget_box(page)
+    if box is None:
+        return False
+    await page.mouse.move(box["x"] + CHECKBOX_INSET, box["y"] + box["height"] / 2)
+    try:
+        await page.mouse.down()
+    finally:
+        await page.mouse.up()
+    return True
+
+
+async def checkbox_already_answered(page: Page) -> bool:
+    """Report whether Turnstile has already filled in its response token."""
+    token = page.locator(TURNSTILE_INPUT)
+    with suppress(PlaywrightError, PlaywrightTimeoutError):
+        if await token.count() > 0:
+            return bool(await token.first.input_value(timeout=TOKEN_READ_TIMEOUT))
+    return False
+
+
+async def challenge_is_gone(page: Page) -> bool:
+    """Confirm the interstitial is really gone and not just between navigations."""
+    if await challenge_present(page):
+        return False
+    await sleep(POLL_INTERVAL)
+    return not await challenge_present(page)
+
+
+async def solve_challenge(page: Page, timer: TimeoutTimer) -> None:
+    """Wait out the interstitial, clicking its checkbox whenever one is offered."""
+    logger.info("Challenge detected, waiting for it to clear...")
+    clicks = 0
+    next_click = 0.0
+    while True:
+        if await challenge_is_gone(page):
+            logger.debug("Challenge cleared.")
+            if clicks:
+                await sleep(min(CLICK_SETTLE, timer.remaining()))
+            return
+
+        if time.perf_counter() >= next_click:
+            landed = False
+            with suppress(PlaywrightError, PlaywrightTimeoutError):
+                landed = not await checkbox_already_answered(
+                    page
+                ) and await click_checkbox(page)
+            if landed:
+                clicks += 1
+                logger.info("Clicked the challenge checkbox (attempt %d).", clicks)
+            next_click = time.perf_counter() + (
+                CLICK_COOLDOWN if landed else PROBE_INTERVAL
+            )
+
+        if timer.remaining() <= 0:
+            break
+        await sleep(POLL_INTERVAL)
+
+    message = "Challenge still present when the request budget ran out"
+    raise TimeoutError(message)

src/consts.py

@@ -1,32 +1,41 @@
 import logging
-import os
-import sys
-from pathlib import Path
 
-from playwright_captcha import CaptchaType
+from pydantic_settings import BaseSettings, SettingsConfigDict
-from playwright_captcha.utils.camoufox_add_init_script.add_init_script import (
-    get_addon_path,
-)
 
-LOG_LEVEL = logging.getLevelNamesMapping()[os.getenv("LOG_LEVEL", "INFO").upper()]
 
-VERSION = os.getenv("VERSION", "unknown").removeprefix("v")
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
-ADDON_PATH = str(Path(get_addon_path()).absolute())
+    log_level: str = "INFO"
-MAX_ATTEMPTS = sys.maxsize
+    version: str = "unknown"
 
-PROXY_SERVER = os.getenv("PROXY_SERVER")
+    proxy_server: str | None = None
-PROXY_USERNAME = os.getenv("PROXY_USERNAME")
+    proxy_username: str | None = None
-PROXY_PASSWORD = os.getenv("PROXY_PASSWORD")
+    proxy_password: str | None = None
 
-HOST = os.getenv("HOST", "0.0.0.0")  # noqa: S104
+    host: str = "0.0.0.0"  # noqa: S104
-PORT = int(os.getenv("PORT", "8191"))
+    port: int = 8191
 
-CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = {
+    block_media: bool = False
-    # Cloudflare
+    return_only_cookies: bool = False
-    CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."],
+    owui_api_key: str | None = None
-}
+    browser_locale: str | None = None
 
-CHALLENGE_TITLES = [
+
-    title for titles in CHALLENGE_TITLES_MAP.values() for title in titles
+settings = Settings()
-]
+
+LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()]
+VERSION = settings.version.removeprefix("v")
+
+PROXY_SERVER = settings.proxy_server
+PROXY_USERNAME = settings.proxy_username
+PROXY_PASSWORD = settings.proxy_password
+
+HOST = settings.host
+PORT = settings.port
+
+BLOCK_MEDIA = settings.block_media
+RETURN_ONLY_COOKIES = settings.return_only_cookies
+
+OWUI_API_KEY = settings.owui_api_key
+BROWSER_LOCALE = settings.browser_locale

src/content.py

@@ -0,0 +1,42 @@
+import base64
+
+from playwright.async_api import Page
+
+from src.models import LinkRequest
+from src.utils import logger
+
+
+async def build_response_content(
+    page: Page,
+    request: LinkRequest,
+    page_request: object,
+    *,
+    challenge_detected: bool,
+    page_html: str | None,
+) -> tuple[str, str]:
+    """Build (content_type, response_content) from the settled page."""
+    if request.return_only_cookies:
+        return "text/html", ""
+
+    if page_request and page_request.headers.get("content-type", "").startswith(
+        "application/pdf"
+    ):
+        return await fetch_pdf_content(page)
+
+    response_content = (
+        page_html
+        if page_html is not None and not challenge_detected
+        else await page.content()
+    )
+    return "text/html", response_content
+
+
+async def fetch_pdf_content(page: Page) -> tuple[str, str]:
+    """Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
+    try:
+        fetch_response = await page.request.fetch(page.url)
+        response_content = base64.b64encode(await fetch_response.body()).decode("ascii")
+    except Exception:
+        logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
+        return "text/html", await page.content()
+    return "application/pdf", response_content

src/endpoints.py

@@ -1,28 +1,35 @@
 import time
 import warnings
-from asyncio import wait_for
 from http import HTTPStatus
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
-from playwright_captcha import CaptchaType
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 
-from src.consts import CHALLENGE_TITLES
+from src.challenge import challenge_present, solve_challenge
+from src.content import build_response_content
 from src.models import (
     HealthcheckResponse,
     LinkRequest,
     LinkResponse,
     Solution,
 )
-from src.utils import CamoufoxDepClass, TimeoutTimer, get_camoufox, logger
+from src.utils import (
+    BrowserDepClass,
+    TimeoutTimer,
+    get_browser,
+    logger,
+    remaining_ms,
+)
 
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
 
 router = APIRouter()
 
-CamoufoxDep = Annotated[CamoufoxDepClass, Depends(get_camoufox)]
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
 
 
 @router.get("/", include_in_schema=False)
@@ -33,7 +40,7 @@ def read_root():
 
 
 @router.get("/health")
-async def health_check(sb: CamoufoxDep):
+async def health_check(sb: BrowserDep):
     """Health check endpoint."""
     health_check_request = await read_item(
         LinkRequest.model_construct(url="https://google.com"),
@@ -50,57 +57,99 @@ async def health_check(sb: CamoufoxDep):
 
 
 @router.post("/v1")
-async def read_item(request: LinkRequest, dep: CamoufoxDep) -> LinkResponse:
+async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     """Handle POST requests."""
     start_time = int(time.time() * 1000)
-
     timer = TimeoutTimer(duration=request.max_timeout)
-
     request.url = request.url.replace('"', "").strip()
-    try:
-        page_request = await dep.page.goto(
-            request.url, timeout=timer.remaining() * 1000
-        )
-        status = page_request.status if page_request else HTTPStatus.OK
-        await dep.page.wait_for_load_state(
-            state="domcontentloaded", timeout=timer.remaining() * 1000
-        )
-        await dep.page.wait_for_load_state(
-            "networkidle", timeout=timer.remaining() * 1000
-        )
 
-        if await dep.page.title() in CHALLENGE_TITLES:
+    await setup_routes(request, dep)
-            logger.info("Challenge detected, attempting to solve...")
+
-            # Solve the captcha
+    try:
-            await wait_for(
+        challenge_detected, page_html, page_request = await _navigate_and_solve(
-                dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
+            dep, request, timer
-                    captcha_container=dep.page,
+        )
-                    captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
+    except (TimeoutError, PlaywrightTimeoutError) as e:
-                    wait_checkbox_attempts=1,
+        logger.error("Timed out while loading the page or solving the challenge")
-                    wait_checkbox_delay=0.5,
-                ),
-                timeout=timer.remaining(),
-            )
-            status = HTTPStatus.OK
-            logger.debug("Challenge solved successfully.")
-    except TimeoutError as e:
-        logger.error("Timed out while solving the challenge")
         raise HTTPException(
             status_code=408,
-            detail="Timed out while solving the challenge",
+            detail="Timed out while loading the page or solving the challenge",
+        ) from e
+    except PlaywrightError as e:
+        logger.error("Could not reach the target: %s", e)
+        raise HTTPException(
+            status_code=502,
+            detail=f"Could not reach the target: {e}",
         ) from e
 
     cookies = await dep.context.cookies()
+    content_type, response_content = await build_response_content(
+        dep.page,
+        request,
+        page_request,
+        challenge_detected=challenge_detected,
+        page_html=page_html,
+    )
+
+    user_agent = (
+        page_request.request.headers.get("user-agent") or "" if page_request else ""
+    )
 
     return LinkResponse(
         message="Success",
         solution=Solution(
-            user_agent=await dep.page.evaluate("navigator.userAgent"),
+            user_agent=user_agent,
             url=dep.page.url,
-            status=status,
+            status=HTTPStatus.OK,
             cookies=cookies,
             headers=page_request.headers if page_request else {},
-            response=await dep.page.content(),
+            response=response_content,
+            content_type=content_type,
         ),
         start_timestamp=start_time,
     )
+
+
+async def setup_routes(request: LinkRequest, dep: BrowserDep) -> None:
+    """Install request routes for media blocking."""
+    if request.block_media:
+
+        async def block_media_route(route) -> None:
+            if route.request.resource_type in ("image", "media", "font"):
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await dep.page.route("**/*", block_media_route)
+
+
+async def _navigate_and_solve(
+    dep: BrowserDep,
+    request: LinkRequest,
+    timer: TimeoutTimer,
+) -> tuple[bool, str | None, object]:
+    """Navigate to the URL, then solve a challenge or wait for network idle."""
+    page_html: str | None = None
+    page_request = await dep.page.goto(request.url, timeout=remaining_ms(timer))
+    await dep.page.wait_for_load_state(
+        state="domcontentloaded", timeout=remaining_ms(timer)
+    )
+
+    if not await challenge_present(dep.page):
+        page_html = await dep.page.content()
+        await _wait_for_networkidle(dep, timer)
+        return False, page_html, page_request
+
+    await solve_challenge(dep.page, timer)
+    await _wait_for_networkidle(dep, timer)
+    return True, page_html, page_request
+
+
+async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:
+    """Wait for network idle, tolerating post-DOM-load stalls."""
+    try:
+        await dep.page.wait_for_load_state("networkidle", timeout=remaining_ms(timer))
+    except PlaywrightTimeoutError:
+        logger.info(
+            "networkidle timed out after domcontentloaded; continuing with loaded page"
+        )

src/models.py

@@ -5,13 +5,17 @@ from http.client import INTERNAL_SERVER_ERROR
 from typing import Any
 
 from playwright.sync_api import Cookie
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic.alias_generators import to_camel
 
 from src import consts
 
+MS_PER_SECOND = 1000
+
 
 class LinkRequest(BaseModel):
+    model_config = {"populate_by_name": True}
+
     cmd: str = Field(
         default="request.get",
         description="Type of request, currently only supports GET requests. This string is purely for compatibility with FlareSolverr.",
@@ -19,8 +23,31 @@ class LinkRequest(BaseModel):
     url: str = Field(pattern=r"^https?://", default="https://")
     max_timeout: int = Field(
         default=60,
-        description="Maximum timeout in seconds for resolving the anti-bot challenge.",
+        alias="maxTimeout",
+        description=(
+            "Maximum timeout for resolving the anti-bot challenge. Values below 1000 "
+            "are treated as seconds; values of 1000 or more as milliseconds, matching "
+            "FlareSolverr's maxTimeout parameter."
+        ),
     )
+    block_media: bool = Field(
+        default=consts.BLOCK_MEDIA,
+        alias="blockMedia",
+        description="Block image, media, and font resources from loading.",
+    )
+    return_only_cookies: bool = Field(
+        default=consts.RETURN_ONLY_COOKIES,
+        alias="returnOnlyCookies",
+        description="Return only cookies, skip the page HTML content in the response.",
+    )
+
+    @field_validator("max_timeout")
+    @classmethod
+    def normalize_max_timeout(cls, value: int) -> int:
+        """Normalize FlareSolverr-style millisecond values to seconds."""
+        if value >= MS_PER_SECOND:
+            return value // MS_PER_SECOND
+        return value
 
 
 class HealthcheckResponse(BaseModel):
@@ -38,6 +65,7 @@ class Solution(BaseModel):
     user_agent: str = ""
     headers: dict[str, Any] = {}
     response: str = ""
+    content_type: str = Field(default="text/html", alias="contentType")
 
 
 class LinkResponse(BaseModel):
@@ -50,7 +78,7 @@ class LinkResponse(BaseModel):
     version: str = consts.VERSION
 
     @classmethod
-    def invalid(cls, url: str):
+    def invalid(cls, url: str) -> LinkResponse:
         """
         Return an invalid LinkResponse with default error values.
 

src/owui.py

@@ -0,0 +1,76 @@
+"""Open WebUI external web loader endpoint: POST /load."""
+
+from __future__ import annotations
+
+from hmac import compare_digest
+from typing import Annotated
+
+import trafilatura
+from fastapi import APIRouter, Depends, Header, HTTPException
+from playwright.async_api import Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from pydantic import BaseModel
+
+from src.consts import OWUI_API_KEY
+from src.utils import BrowserDepClass, get_browser, logger
+
+router = APIRouter(tags=["Open WebUI"])
+
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+
+class LoadRequest(BaseModel):
+    urls: list[str]
+
+
+class LoadResult(BaseModel):
+    page_content: str
+    metadata: dict[str, str]
+
+
+def require_auth(authorization: Annotated[str | None, Header()] = None) -> None:
+    """Enforce a bearer token on /load when OWUI_API_KEY is set."""
+    if not OWUI_API_KEY:
+        return
+    if authorization is None or not compare_digest(
+        authorization.encode(), f"Bearer {OWUI_API_KEY}".encode()
+    ):
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+async def _extract_content(page: Page) -> str:
+    """Return the page's main article text, falling back to visible text."""
+    article = trafilatura.extract(await page.content())
+    if article:
+        return article
+    result = await page.locator("body").inner_text()
+    return "\n".join(line.strip() for line in result.splitlines() if line.strip())
+
+
+@router.post("/load", response_model=list[LoadResult])
+async def load_urls(
+    request: LoadRequest,
+    _auth: Annotated[None, Depends(require_auth)],
+    dep: BrowserDep,
+) -> list[LoadResult]:
+    """
+    Fetch URLs through the anti-bot browser and return their text content.
+
+    Each URL is fetched sequentially; a failing URL yields empty
+    page_content so Open WebUI's RAG pipeline degrades gracefully.
+    """
+    results: list[LoadResult] = []
+    for url in request.urls:
+        try:
+            await dep.page.goto(url, timeout=60_000)
+            await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            try:
+                await dep.page.wait_for_load_state("networkidle", timeout=15_000)
+            except PlaywrightTimeoutError:
+                logger.debug("networkidle timed out for %s; extracting anyway", url)
+            content = await _extract_content(dep.page)
+        except Exception as exc:
+            logger.warning("Failed to load %s: %s", url, exc)
+            content = ""
+        results.append(LoadResult(page_content=content, metadata={"source": url}))
+    return results

src/utils.py

@@ -3,19 +3,14 @@ import time
 from collections.abc import AsyncGenerator
 from typing import Annotated, NamedTuple, cast
 
-from camoufox import AsyncCamoufox
 from fastapi import Header
+from invisible_playwright.async_api import InvisiblePlaywright
 from playwright.async_api import Browser, BrowserContext, Page
-from playwright_captcha import (
-    ClickSolver,
-    FrameworkType,
-)
 from pydantic import BaseModel, Field
 
 from src.consts import (
-    ADDON_PATH,
+    BROWSER_LOCALE,
     LOG_LEVEL,
-    MAX_ATTEMPTS,
     PROXY_PASSWORD,
     PROXY_SERVER,
     PROXY_USERNAME,
@@ -44,13 +39,20 @@ class TimeoutTimer(BaseModel):
         return max(0, self.duration - (time.perf_counter() - self.start_time))
 
 
-class CamoufoxDepClass(NamedTuple):
+MIN_WAIT_MS = 1.0
+
+
+def remaining_ms(timer: TimeoutTimer) -> float:
+    """Milliseconds left, never 0 - Playwright reads that as no timeout at all."""
+    return max(MIN_WAIT_MS, timer.remaining() * 1000)
+
+
+class BrowserDepClass(NamedTuple):
     page: Page
-    solver: ClickSolver
     context: BrowserContext
 
 
-async def get_camoufox(
+async def get_browser(
     x_proxy_server: Annotated[
         str | None,
         Header(
@@ -70,8 +72,8 @@ async def get_camoufox(
             alias="X-Proxy-Password",
         ),
     ] = None,
-) -> AsyncGenerator[CamoufoxDepClass]:
+) -> AsyncGenerator[BrowserDepClass]:
-    """Get Camoufox instance."""
+    """Get InvisiblePlaywright browser instance."""
     header_server = x_proxy_server
     header_username = x_proxy_username
     header_password = x_proxy_password
@@ -91,26 +93,19 @@ async def get_camoufox(
             "password": PROXY_PASSWORD,
         }
 
-    async with AsyncCamoufox(
+    async with InvisiblePlaywright(
-        main_world_eval=True,
-        addons=[ADDON_PATH],
-        geoip=True,
-        proxy=proxy_config,
-        locale="en-US",
         headless=True,
+        proxy=proxy_config,
         humanize=True,
-        i_know_what_im_doing=True,
+        locale=BROWSER_LOCALE or "auto",
-        config={"forceScopeAccess": True},  # add this when creating Camoufox instance
+        extra_prefs={
-        disable_coop=True,  # add this when creating Camoufox instance
+            "devtools.jsonview.enabled": False,
+            "browser.tabs.remote.useCrossOriginOpenerPolicy": False,
+            "browser.tabs.remote.useCrossOriginEmbedderPolicy": False,
+        },
     ) as browser_raw:
-        # Cast to Browser since AsyncCamoufox always returns a Browser, not BrowserContext
+        # InvisiblePlaywright yields a Browser instance
         browser = cast("Browser", browser_raw)
         context = await browser.new_context()
         page = await context.new_page()
-        async with ClickSolver(
+        yield BrowserDepClass(page, context)
-            framework=FrameworkType.CAMOUFOX,
-            page=page,
-            max_attempts=MAX_ATTEMPTS,
-            attempt_delay=1,
-        ) as solver:
-            yield CamoufoxDepClass(page, solver, context)

tests/main_test.py

@@ -1,12 +1,20 @@
+import base64
 from http import HTTPStatus
 from json import JSONDecodeError
+from unittest.mock import AsyncMock, MagicMock
 
-import httpx
+import httpx2
 import pytest
+from fastapi import HTTPException
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
+from src.challenge import CF_INTERSTITIAL_INDICATORS_SELECTORS
+from src.endpoints import read_item
 from src.models import LinkRequest
+from src.utils import BrowserDepClass, TimeoutTimer, remaining_ms
 
 client = TestClient(app)
 
@@ -27,7 +35,7 @@ def test_bypass(website: str):
 
     This test is skipped if the website is not reachable or does not have cloudflare/DDOS-GUARD.
     """
-    test_request = httpx.get(
+    test_request = httpx2.get(
         website,
     )
     if (
@@ -44,10 +52,44 @@ def test_bypass(website: str):
 
     response = client.post(
         "/v1",
-        json=LinkRequest.model_construct(url=website, cmd="request.get").model_dump(),
+        json=LinkRequest.model_construct(
+            url=website, cmd="request.get", max_timeout=60
+        ).model_dump(),
     )
 
     assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    assert "_cf_chl_opt" not in solution["response"]
+    assert "__cf_chl" not in solution["url"]
+
+
+def test_json_api():
+    """
+    JSON APIs must return 200, not crash on the UA evaluate.
+
+    Firefox renders application/json in a built-in viewer whose CSP blocks
+    Playwright's eval-based evaluate() (issue #394). The browser must be
+    launched with the viewer disabled so /v1 works and returns the raw JSON.
+    """
+    url = "https://api.ipify.org?format=json"
+    test_request = httpx2.get(url)
+    if test_request.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR:
+        pytest.skip(
+            f"Skipping JSON API test - upstream error ({test_request.status_code})"
+        )
+
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=url, cmd="request.get").model_dump(),
+    )
+
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping JSON API test - timed out (upstream issue)")
+
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    assert solution["userAgent"]
+    assert '"ip"' in solution["response"]
 
 
 def test_health_check():
@@ -59,3 +101,223 @@ def test_health_check():
     """
     response = client.get("/health")
     assert response.status_code == HTTPStatus.OK
+
+
+def test_pdf_handling():
+    """Tests that PDF URLs return the raw PDF bytes, not the Firefox viewer HTML."""
+    pdf_url = "https://mondaymandala.com/wp-content/uploads/Mickey-And-Minnie-Mouse-Holding-An-Easter-Egg-Basket-Coloring-Page-For-Kids.pdf"
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=pdf_url, cmd="request.get").model_dump(),
+    )
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping PDF test - timed out (upstream issue)")
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    if solution.get("contentType") != "application/pdf":
+        pytest.skip(
+            "Skipping PDF test - PDF bytes could not be fetched (upstream issue)"
+        )
+    assert solution["response"]  # non-empty base64
+
+    decoded = base64.b64decode(solution["response"])
+    assert decoded[:5] == b"%PDF-"
+
+
+@pytest.mark.parametrize(
+    ("payload", "expected"),
+    [
+        ({"max_timeout": 60}, 60),  # native API: seconds
+        ({"maxTimeout": 60}, 60),  # FlareSolverr alias, seconds-range value
+        ({"maxTimeout": 60000}, 60),  # FlareSolverr alias: milliseconds
+        ({"maxTimeout": 55000}, 55),
+        ({"maxTimeout": 1000}, 1),
+        ({}, 60),  # default
+    ],
+)
+def test_max_timeout_normalization(payload: dict, expected: int):
+    """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds."""
+    request = LinkRequest(url="https://example.com", **payload)
+    assert request.max_timeout == expected
+
+
+def fake_dep(
+    *,
+    fail_states: set[str] | None = None,
+    challenged: bool = False,
+    marker_counts: list[int] | None = None,
+    widget_box: dict[str, float] | None = None,
+    user_agent: str | None = "UnitTestBrowser/1.0",
+) -> BrowserDepClass:
+    """Build a browser dependency pair backed by mocks."""
+    page = AsyncMock()
+    page.url = "https://example.test/login"
+    page.goto.return_value = MagicMock(
+        status=HTTPStatus.OK,
+        headers={"content-type": "text/html"},
+        request=MagicMock(headers={"user-agent": user_agent} if user_agent else {}),
+    )
+    page.title.return_value = "Login"
+    page.content.return_value = "<html><title>Login</title></html>"
+    remaining = list(marker_counts or [])
+
+    def count_for(selector: str) -> int:
+        """Answer the marker check from the script, else from `challenged`."""
+        if selector not in CF_INTERSTITIAL_INDICATORS_SELECTORS or not remaining:
+            return 1 if challenged else 0
+        return remaining.pop(0) if len(remaining) > 1 else remaining[0]
+
+    def locator(selector: str) -> MagicMock:
+        handle = MagicMock()
+        handle.count = AsyncMock(side_effect=lambda: count_for(selector))
+        handle.first.bounding_box = AsyncMock(return_value=widget_box)
+        handle.first.input_value = AsyncMock(return_value="")
+        return handle
+
+    page.locator = MagicMock(side_effect=locator)
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        """Fail the wait when asked for a configured state."""
+        if state in (fail_states or set()):
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+
+    context = AsyncMock()
+    context.cookies.return_value = []
+    return BrowserDepClass(page=page, context=context)
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_after_domcontentloaded_returns_content():
+    """Pages that never go idle after DOM load must still return their content."""
+    dep = fake_dep(fail_states={"networkidle"})
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        dep,
+    )
+
+    assert response.status == "ok"
+    assert response.solution.response == "<html><title>Login</title></html>"
+
+
+@pytest.mark.asyncio
+async def test_domcontentloaded_timeout_returns_408():
+    """Fatal timeouts during initial page load still return a controlled 408."""
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login"),
+            fake_dep(fail_states={"domcontentloaded"}),
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT
+
+
+@pytest.mark.asyncio
+async def test_unreachable_host_is_a_502_not_a_500():
+    """An upstream we cannot reach is a gateway failure, never a Byparr crash."""
+    dep = fake_dep()
+    dep.page.goto.side_effect = PlaywrightError("Page.goto: NS_ERROR_UNKNOWN_HOST")
+
+    with pytest.raises(HTTPException) as exc:
+        await read_item(LinkRequest(url="https://nope.invalid/"), dep)
+
+    assert exc.value.status_code == HTTPStatus.BAD_GATEWAY
+
+
+@pytest.mark.asyncio
+async def test_status_is_always_ok_like_flaresolverr():
+    """FlareSolverr hardcodes 200 because Selenium cannot report the real code."""
+    dep = fake_dep()
+    dep.page.goto.return_value = MagicMock(
+        status=HTTPStatus.FORBIDDEN,
+        headers={"content-type": "text/html"},
+        request=MagicMock(headers={"user-agent": "UnitTestBrowser/1.0"}),
+    )
+
+    response = await read_item(LinkRequest(url="https://example.test/login"), dep)
+
+    assert response.solution.status == HTTPStatus.OK
+
+
+def test_exhausted_budget_never_disables_playwright_timeouts():
+    """Playwright reads timeout=0 as no timeout at all, so the floor must hold."""
+    spent = TimeoutTimer(duration=0)
+
+    assert spent.remaining() == 0
+    assert remaining_ms(spent) > 0
+
+
+@pytest.mark.asyncio
+async def test_missing_user_agent_header_is_not_a_500():
+    """A request without a user-agent header degrades to empty, never a 500 (#394)."""
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        fake_dep(user_agent=None),
+    )
+
+    assert response.status == "ok"
+    assert response.solution.user_agent == ""
+
+
+@pytest.mark.asyncio
+async def test_checkbox_is_clicked_while_the_challenge_is_up():
+    """A measurable widget gets a humanised press, not a raw synthetic click."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 1, 0],
+        widget_box={"x": 100.0, "y": 200.0, "width": 300.0, "height": 60.0},
+    )
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login", max_timeout=5), dep
+    )
+
+    assert response.status == "ok"
+    dep.page.mouse.move.assert_awaited_once_with(125.0, 230.0)
+    dep.page.mouse.down.assert_awaited_once()
+    dep.page.mouse.up.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_challenge_that_clears_on_its_own_is_never_clicked():
+    """A challenge is over when its markup goes, and until then we keep our hands off."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 0],
+        widget_box={"x": 100.0, "y": 200.0, "width": 300.0, "height": 60.0},
+    )
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login", max_timeout=5), dep
+    )
+
+    assert response.status == "ok"
+    dep.page.mouse.down.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_challenge_that_never_clears_returns_408():
+    """A challenge still up when the budget runs out is a timeout, not a 500."""
+    dep = fake_dep(challenged=True, marker_counts=[1])
+
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login", max_timeout=2), dep
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT
+
+
+@pytest.mark.asyncio
+async def test_marker_vanishing_mid_navigation_is_not_a_solved_challenge():
+    """The marker drops out between challenge rounds; one clear read proves nothing."""
+    dep = fake_dep(challenged=True, marker_counts=[1, 0, 1])
+
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login", max_timeout=2), dep
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT

tests/owui_test.py

@@ -0,0 +1,115 @@
+from http import HTTPStatus
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from starlette.testclient import TestClient
+
+from main import app
+from src.owui import LoadRequest, load_urls
+from src.utils import BrowserDepClass
+
+client = TestClient(app)
+
+
+def test_owui_load_basic():
+    """/load returns one result per URL with the expected shape."""
+    response = client.post("/load", json={"urls": ["https://example.com"]})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"]
+    assert results[0]["metadata"] == {"source": "https://example.com"}
+
+
+def test_owui_load_multiple_urls():
+    """/load returns one result per URL, in order."""
+    urls = ["https://example.com", "https://example.org"]
+    response = client.post("/load", json={"urls": urls})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert [r["metadata"]["source"] for r in results] == urls
+
+
+def test_owui_load_invalid_url_graceful():
+    """Unreachable URLs yield empty page_content instead of an error."""
+    response = client.post(
+        "/load", json={"urls": ["https://this-domain-does-not-exist-12345.invalid"]}
+    )
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"] == ""
+
+
+@pytest.mark.parametrize(
+    "headers",
+    [None, {"Authorization": "Bearer wrong-key"}],
+)
+def test_owui_load_rejects_missing_or_wrong_key(headers):
+    """/load returns 401 without a valid bearer token when a key is set."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load", json={"urls": ["https://example.com"]}, headers=headers
+        )
+        assert response.status_code == HTTPStatus.UNAUTHORIZED
+
+
+def test_owui_load_accepts_valid_key():
+    """/load succeeds with the configured bearer token."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load",
+            json={"urls": ["https://example.com"]},
+            headers={"Authorization": "Bearer test-secret-key"},
+        )
+        assert response.status_code == HTTPStatus.OK
+
+
+ARTICLE_HTML = """<html><head><title>Test</title></head><body>
+<article><h1>Example Title</h1><p>This is the main article body with enough words for
+trafilatura to consider it real content rather than boilerplate.</p></article>
+<nav><a href="/x">nav link</a></nav>
+</body></html>"""
+
+
+def fake_dep(*, html: str = ARTICLE_HTML) -> BrowserDepClass:
+    """Browser dependency whose page loads HTML but never reaches networkidle."""
+    page = AsyncMock()
+    page.goto.return_value = MagicMock()
+    page.content.return_value = html
+    page.locator = MagicMock()
+    page.locator.return_value.inner_text = AsyncMock(
+        return_value="line one\n\nline two"
+    )
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        if state == "networkidle":
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+    return BrowserDepClass(page=page, context=AsyncMock())
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_still_extracts_content():
+    """A page that never reaches networkidle still yields its article text."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]), None, fake_dep()
+    )
+    assert results[0].page_content == (
+        "Example TitleThis is the main article body with enough words for "
+        "trafilatura to consider it real content rather than boilerplate."
+    )
+
+
+@pytest.mark.asyncio
+async def test_extraction_falls_back_to_innertext():
+    """Pages trafilatura cannot score fall back to the rendered innerText."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]),
+        None,
+        fake_dep(html="<html><body></body></html>"),
+    )
+    assert results[0].page_content == "line one\nline two"