Add sponsor badge to README

Added a sponsor badge to the README.

.devcontainer/devcontainer.json

@@ -9,7 +9,11 @@
   "runArgs": ["--security-opt", "label=disable", "--privileged"],
   "customizations": {
     "vscode": {
-      "extensions": ["charliermarsh.ruff", "ms-python.python"],
+      "extensions": [
+        "charliermarsh.ruff",
+        "ms-python.python",
+        "tamasfe.even-better-toml"
+      ],
       "settings": {
         "python.venvPath": "./venv"
       }
@@ -17,7 +21,7 @@
   },
   "postCreateCommand": "uv sync --group test",
   "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+    "ghcr.io/devcontainers/features/docker-in-docker:4": {
       "moby": false
     }
   }

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# Supported funding model platforms
+
+github: [ThePhaseless]

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -13,6 +13,8 @@ body:
         - label: Checked if such issue already exists
         - label: Checked other websites with Cloudflare Turnstile
         - label: I am able to access the webpage from my browser
+        - label: (Docker only) Checked that the problem does also occur on main tag
+
   - type: checkboxes
     attributes:
       label: "If checked other websites with Cloudflare Turnstile:"
@@ -20,6 +22,13 @@ body:
         - label: Other websites do work.
         - label: Other websites do not work.
 
+  - type: checkboxes
+    attributes:
+      label: "The issue is still present in the latest main tag:"
+      options:
+        - label: "Yes"
+        - label: "No"
+
   - type: input
     id: docker-host
     attributes:

.github/workflows/docker-cleanup.yml

@@ -0,0 +1,107 @@
+name: Cleanup PR Docker Images
+
+on:
+  pull_request:
+    types: [closed]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Delete PR Docker images
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const owner = context.repo.owner.toLowerCase();
+            const repo = context.repo.repo.toLowerCase();
+            const prNumber = context.payload.pull_request.number;
+
+            // Get the package
+            const packageName = `${repo}`;
+
+            console.log(`Cleaning up images for PR #${prNumber}`);
+
+            // Check if owner is an org or user
+            let isOrg = false;
+            try {
+              await github.rest.orgs.get({ org: owner });
+              isOrg = true;
+              console.log(`Repository owner is an organization`);
+            } catch (error) {
+              console.log(`Repository owner is a user account`);
+            }
+
+            try {
+              // Get all versions of the package
+              let versions;
+              if (isOrg) {
+                versions = await github.rest.packages.getAllPackageVersionsForPackageOwnedByOrg({
+                  package_type: 'container',
+                  package_name: packageName,
+                  org: owner,
+                  per_page: 100
+                });
+              } else {
+                versions = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser({
+                  package_type: 'container',
+                  package_name: packageName,
+                  username: owner,
+                  per_page: 100
+                });
+              }
+
+              // Filter versions that match this PR (pr-123, pr-123-amd64, pr-123-arm64)
+              const prVersions = versions.data.filter(version => {
+                const tags = version.metadata?.container?.tags || [];
+                // Match tags like: pr-123, pr-123-amd64, pr-123-arm64
+                return tags.some(tag =>
+                  tag === `pr-${prNumber}` ||
+                  tag === `pr-${prNumber}-amd64` ||
+                  tag === `pr-${prNumber}-arm64`
+                );
+              });
+
+              console.log(`Found ${prVersions.length} image version(s) to delete`);
+
+              // Delete each version
+              for (const version of prVersions) {
+                console.log(`Deleting version ${version.id} with tags: ${version.metadata?.container?.tags?.join(', ')}`);
+                try {
+                  if (isOrg) {
+                    await github.rest.packages.deletePackageVersionForOrg({
+                      package_type: 'container',
+                      package_name: packageName,
+                      org: owner,
+                      package_version_id: version.id
+                    });
+                  } else {
+                    await github.rest.packages.deletePackageVersionForUser({
+                      package_type: 'container',
+                      package_name: packageName,
+                      username: owner,
+                      package_version_id: version.id
+                    });
+                  }
+                  console.log(`✓ Deleted version ${version.id}`);
+                } catch (error) {
+                  console.error(`Failed to delete version ${version.id}:`, error.message);
+                }
+              }
+
+              console.log('Cleanup completed');
+            } catch (error) {
+              if (error.status === 404) {
+                console.log('No package found - nothing to clean up');
+              } else {
+                console.error('Error during cleanup:', error.message);
+                throw error;
+              }
+            }

.github/workflows/docker-publish.yml

@@ -9,11 +9,29 @@ on:
   schedule:
     - cron: "25 0 * * *"
   push:
-    branches: ["*"]
+    branches: ["main"]
     # Publish semver tags as releases.
     tags: ["v*.*.*"]
+    paths:
+      - "src/**"
+      - "main.py"
+      - "tests/**"
+      - "Dockerfile"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "compose.yaml"
+      - ".github/workflows/docker-publish.yml"
   pull_request:
     branches: ["main"]
+    paths:
+      - "src/**"
+      - "main.py"
+      - "tests/**"
+      - "Dockerfile"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "compose.yaml"
+      - ".github/workflows/docker-publish.yml"
   workflow_dispatch:
 
 env:
@@ -36,24 +54,28 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Test
         id: test
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64
-          cache-from: type=gha,scope=x64
+          cache-from: type=gha,scope=amd64
           pull: true
-          cache-to: type=gha,mode=max,scope=x64
+          cache-to: type=gha,mode=max,scope=amd64
           target: test
+          build-args: |
+            GITHUB_BUILD=true
 
   build:
     needs: test
+    # Skip build for Renovate PRs
+    if: github.event.pull_request.user.login != 'renovate[bot]'
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -66,26 +88,31 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       - name: Prepare variables
         id: vars
         run: |
           SURFIX=$(echo ${{ matrix.platform }} | cut -d'/' -f2)
           echo "SURFIX=$SURFIX" >> $GITHUB_OUTPUT
-          # Generate a unique local tag for the image
+
-          echo "LOCAL_TAG=${{ github.sha }}-$SURFIX" >> $GITHUB_OUTPUT
+          # Use PR number for PRs, SHA for everything else
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "LOCAL_TAG=pr-${{ github.event.pull_request.number }}-$SURFIX" >> $GITHUB_OUTPUT
+          else
+            echo "LOCAL_TAG=${{ github.sha }}-$SURFIX" >> $GITHUB_OUTPUT
+          fi
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       # Set up BuildKit Docker container builder
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Log into registry
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -94,24 +121,24 @@ jobs:
       # Extract metadata (tags, labels) for Docker
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           tags: type=raw,value=${{ steps.vars.outputs.LOCAL_TAG }}
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      # Build and export Docker image for each platform (without pushing)
+      # Build and push Docker image for each platform
       - name: Build Docker image
         id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           pull: true
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ matrix.platform }}
-          cache-from: type=gha,scope=${{ matrix.platform }}
+          cache-from: type=gha,scope=${{ steps.vars.outputs.SURFIX }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.SURFIX }}
           build-args: |
             GITHUB_BUILD=true
             VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.sha }}
@@ -119,7 +146,6 @@ jobs:
   merge-and-push:
     needs: build
     runs-on: ubuntu-latest
-    if: github.event_name != 'pull_request'
     permissions:
       contents: read
       packages: write
@@ -127,19 +153,19 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       # Install the cosign tool
       - name: Install cosign
-        uses: sigstore/cosign-installer@v3
+        uses: sigstore/cosign-installer@v4.1.2
 
       # Set up Docker Buildx
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Log into registry
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -148,7 +174,7 @@ jobs:
       # Extract Docker metadata for tagging
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           tags: |
             type=ref,event=branch
@@ -160,6 +186,7 @@ jobs:
 
       # Create manifest lists and push
       - name: Create and push manifest lists
+        id: manifests
         run: |
           TAGS="${{ steps.meta.outputs.tags }}"
           args=""
@@ -178,15 +205,30 @@ jobs:
 
           echo $args
 
-          docker buildx imagetools create $args \
+          # Use PR-based tags for PRs, SHA-based tags for everything else
-          ${image}:${{github.sha}}-amd64 \
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
-          ${image}:${{github.sha}}-arm64
+            docker buildx imagetools create $args \
+            ${image}:pr-${{ github.event.pull_request.number }}-amd64 \
+            ${image}:pr-${{ github.event.pull_request.number }}-arm64
+          else
+            docker buildx imagetools create $args \
+            ${image}:${{github.sha}}-amd64 \
+            ${image}:${{github.sha}}-arm64
+          fi
 
-      # Sign the manifest
+          # All tags created above alias a single manifest list; capture its digest
-      - name: Sign the manifests
+          # so the signature is bound to the image bytes, not a mutable tag.
+          # Tags from metadata-action are full references (image:tag), one per line,
+          # so take the first line rather than splitting on spaces.
+          FIRST_TAG=$(printf '%s' "$TAGS" | head -n1)
+          DIGEST=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$FIRST_TAG")
+          echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
+
+      # Sign the manifest list by digest — every consumer tag aliases this digest
+      - name: Sign the manifest list by digest
         env:
-          TAGS: ${{ steps.meta.outputs.tags }}
+          IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          DIGEST: ${{ steps.manifests.outputs.DIGEST }}
         run: |
-          for TAG in $TAGS; do
+          image=${IMAGE,,}
-            cosign sign --yes $TAG
+          cosign sign --yes ${image}@${DIGEST}
-          done

AGENTS.md

@@ -0,0 +1,37 @@
+# Copilot Instructions
+
+## Project overview
+
+- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using invisible_playwright.
+- Entry point: main app in main.py; routes and request flow in src/endpoints.py, challenge handling in src/challenge.py, response bodies in src/content.py, models in src/models.py.
+- Browser lifecycle is owned by get_browser() in src/utils.py, which yields a page and context for each request.
+
+## Architecture and data flow
+
+- Request flow: POST /v1 -> read_item() -> page.goto() -> wait for load states -> detect the interstitial -> click its checkbox until it clears -> return LinkResponse.
+- Challenge detection uses detect_cloudflare_challenge() from playwright_captcha; the challenge is over when its markup goes, not when a solver says so.
+- Health check hits /v1 internally with <https://google.com> and fails if status is not OK.
+- Logging: LogRequest middleware logs only POST /v1 timing and outcome; other paths pass through.
+
+## Key modules and patterns
+
+- Models use Pydantic v2 with camelCase aliasing for responses (see src/models.py).
+- LinkResponse.invalid() is the standard error response shape; keep fields consistent with FlareSolverr style.
+- get_camoufox() constructs AsyncCamoufox with addons and optional proxy config from env vars.
+
+## Config and environment
+
+- Core env vars in src/consts.py: HOST, PORT, PROXY_SERVER, PROXY_USERNAME, PROXY_PASSWORD, LOG_LEVEL, VERSION.
+- VERSION strips leading "v" for tag-style values.
+
+## Developer workflows
+
+- Local run: uv sync && uv run main.py
+- Init mode: uv run main.py --init (pre-warms health check via browser setup)
+- Tests: uv sync --group test && uv run pytest --retries 5
+- Docker troubleshooting: docker build --target test .
+
+## Tests and external dependencies
+
+- tests/main_test.py calls real websites; tests are network-dependent and may be skipped based on upstream status.
+- HTTP client tests use starlette.testclient + httpx; avoid mocking unless needed for local-only changes.

Dockerfile

@@ -1,42 +1,62 @@
-FROM debian:stable-slim AS base
+# Ubuntu is required by playwright.
-ENV HOME=/root
+# Pin to 24.04 LTS: ubuntu:latest floats to 26.04, which Playwright 1.58
+# cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch).
+FROM ubuntu:24.04 AS base
 
-ARG GITHUB_BUILD=false \
+ARG GITHUB_BUILD=false
-    VERSION
 
 ENV GITHUB_BUILD=${GITHUB_BUILD}\
-    VERSION=${VERSION}\
-    DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
     # prevents python creating .pyc files
     PYTHONDONTWRITEBYTECODE=1 \
     UV_LINK_MODE=copy \
-    PATH="${HOME}/.local/bin:$PATH"
+    PORT=8191 \
+    XDG_CACHE_HOME=/cache \
+    HOME=/home/byparr
 
-WORKDIR /app
+RUN apt-get update &&\
-
+    apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
-RUN apt update && apt -y upgrade && apt install -y curl
+    apt-get clean &&\
-
+    rm -rf /var/lib/apt/lists/*
-ADD https://astral.sh/uv/install.sh install.sh
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
-RUN sh install.sh && uv --version
-RUN uvx playwright install-deps firefox && uvx camoufox fetch
 
 FROM base AS devcontainer
-RUN apt install -y git
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends git &&\
+    uvx playwright install-deps firefox &&\
+    uvx --from git+https://github.com/feder-cr/invisible_playwright.git python -m invisible_playwright fetch &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 ENTRYPOINT [ "sleep", "infinity" ]
 
-
 FROM base AS app
+WORKDIR /app
+
 COPY pyproject.toml uv.lock ./
-RUN --mount=type=cache,target=${HOME}/.cache/uv uv sync
+RUN mkdir -p /cache &&\
+    uv sync &&\
+    uv run python -m invisible_playwright fetch &&\
+    apt-get update &&\
+    uv run playwright install-deps firefox &&\
+    uv cache clean &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
+RUN mkdir -p /home/byparr &&\
+    chmod -R o+rX /app &&\
+    chmod -R a+rwX /cache /home/byparr
+
 FROM app AS test
-RUN --mount=type=cache,target=${HOME}/.cache/uv uv sync --group test
+RUN \
-RUN uv run pytest --retries 3 -n auto
+    uv sync --group test &&\
+    uv run pytest -rs --retries 5
 
 FROM app
-EXPOSE 8191
+ARG VERSION
-HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
+ENV VERSION=${VERSION}
-ENTRYPOINT ["uv", "run", "main.py"]
+USER 1000
+EXPOSE $PORT
+HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health"
+ENTRYPOINT ["tini", "--", "/app/.venv/bin/python", "main.py"]

README.md

@@ -1,44 +1,29 @@
-# Byparr
+# Byparr [![Sponsor](https://img.shields.io/badge/Sponsor-%E2%9D%A4-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/ThePhaseless)
 
-Built with [camoufox](https://camoufox.com/) and [FastAPI](https://fastapi.tiangolo.com), this project aims to mimic [FlareSolverr's](https://github.com/FlareSolverr/FlareSolverr) API and functionality of providing you with http cookies and headers for websites protected with anti-bot protections.
+<p align="center">
+   <img src="icon/logo-byparr.svg" alt="Byparr logo" width="120" />
+</p>
 
 > [!IMPORTANT]
 > This software does not **guarantee** (only greatly increases the chance) that any challenge will be bypassed. While this tool passes the initial browser check, Cloudflare and other captcha providers likely require valid network traffic originating from the user’s public IP address to mark a connection as legitimate. If any website does not pass the challenge, please run troubleshooting steps and check if other websites work before you create an GitHub issue.
 
-> [!IMPORTANT]
-> Support for NAS devices (like Synology) is minimal. Please report issues, but do not expect it to be fixed quickly. The only ARM device I have is a free Ampere Oracle VM, so I can only test ARM support on that. See [#22](https://github.com/ThePhaseless/Byparr/issues/22) and [#3](https://github.com/ThePhaseless/Byparr/issues/3)
-
-> [!NOTE]
-> Thanks to FastAPI implementation, now you can also see the API documentation at `/docs` or `/` (redirect to `/docs`) endpoints.
-
-## Troubleshooting
-
-### Docker
-
-1. Clone repo to the host that has issues with Byparr.
-2. Run `docker build --target test .`
-3. Depending of the build success:
-   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
-   2. If build fails, try troubleshooting on another host/using other method
-
-### Local
-
-1. Download [uv](https://docs.astral.sh/uv/getting-started/installation/)
-2. Download dependencies using `uv sync --group test`
-3. Run tests with `uv run pytest --retries 3` (You can add `-n auto` for parallelization)
-4. If you see any `F` character in terminal, that means test failed even after retries.
-5. Depending of the test success:
-   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
-   2. If test fails, try troubleshooting on another host/using other method
-
 ## Options
 
 | Environment Variable | Default   | Description                                                                                                                                                       |
 | -------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `HOST`               | `0.0.0.0` | Host address to bind the server to. Use `0.0.0.0` to bind to all IPv4 interfaces, `::` for all IPv6 interfaces, or `127.0.0.1`/`localhost` for local access only. |
+| `PORT`               | `8191`    | Port to bind the server to.                                                                                                                                       |
 | `PROXY_SERVER`       | None      | Proxy to use in format: `protocol://host:port`.                                                                                                                   |
-| `PROXY_USERNAME`     | None      |                                                                                                                                                                   |
+| `PROXY_USERNAME`     | None      | Username for proxy authentication.                                                                                                                                |
-| `PROXY_PASSWORD`     | None      |                                                                                                                                                                   |
+| `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
+| `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |
+| `BROWSER_LOCALE`     | None      | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. |
+
+#### Browser language
+
+Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag like `en-US`, `de-DE`, `fr-FR`, `pl-PL`, or `zh-CN` to fix the browser's language and `Accept-Language` header. When unset, Byparr derives the locale from the egress country (e.g. a French proxy → `fr-FR`), keeping the browser language consistent with the exit IP.
+
+Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`.
 
 ## Proxy Recommendation
 
@@ -48,30 +33,90 @@ Recently I've partnered with a _new in town_ proxy service - ProxyBase - to offe
 
 - `v*.*.*`/`latest` - Releases considered stable
 - `main` - Latest release from main branch (untested)
+- `pr-{number}` - Pull request images for testing (automatically cleaned up when PR closes)
 
 ## Usage
 
-### Docker Compose
+> [!IMPORTANT]
+> Support for NAS devices (like Synology) is minimal. Please report issues, but do not expect it to be fixed quickly. The only ARM device I have is a free Ampere Oracle VM, so I can only test ARM support on that. See [#22](https://github.com/ThePhaseless/Byparr/issues/22) and [#3](https://github.com/ThePhaseless/Byparr/issues/3)
 
-See `compose.yaml`
+### Docker Compose setup
 
-### Docker
+1. Review settings in `compose.yaml`.
+2. Start the service:
 
 ```bash
-docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
+docker compose up -d
 ```
 
-### Local
+### Docker install
+
+1. Pull and run the image:
+
+   ```bash
+   docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
+   ```
+
+2. Optional: set env vars using `-e` or `--env-file`.
+
+### Local install
+
+1. Install ([or update when Python version changes](https://github.com/astral-sh/uv/issues/17887)) [uv](https://docs.astral.sh/uv/getting-started/installation/).
+2. Clone this repo - `git clone https://github.com/ThePhaseless/Byparr`
+3. Run `uv run main.py`
+4. Enjoy!
+
+### API Docs
+
+Once running, open:
+
+- `http://localhost:8191/docs`
+- `http://localhost:8191/` (redirects to `/docs`)
+
+### Open WebUI Integration
+
+Byparr can serve as an external web loader for [Open WebUI](https://github.com/open-webui/open-webui), allowing it to fetch web content through Byparr's anti-bot bypassing capabilities.
+
+Configure Open WebUI with these environment variables:
 
 ```bash
-uv sync && uv run main.py
+WEB_LOADER_ENGINE=external
+EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load
+EXTERNAL_WEB_LOADER_API_KEY=your-secret-key  # Optional, must match OWUI_API_KEY
 ```
 
-## Need help with / TODO
+The `/load` endpoint accepts `POST` requests with `{"urls": ["https://..."]}` and returns extracted text content for RAG pipelines.
 
-- [x] Slimming container (only ~1.11 GB now!)
+## Troubleshooting
-- [x] Add more anti-bot challenges
+
-- [x] Add doc strings
+### Docker troubleshooting
-- [x] Implement versioning
+
-- [x] Proxy support
+1. Clone repo to the host that has issues with Byparr.
-- [x] Add more architectures support
+2. Run `docker build --target test .`
+3. Depending of the build success:
+   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
+   2. If build fails, try troubleshooting on another host/using other method
+
+#### Proxmox OCI / LXC browser launch errors
+
+If you are running Byparr as an OCI container in Proxmox (or another LXC-based setup) and see a `FileNotFoundError` from `multiprocessing.synchronize`/`camoufox` when processing requests, increase the service's shared memory in `compose.yaml`:
+
+```yaml
+services:
+  byparr:
+    shm_size: 512mb
+    stdin_open: true
+    tty: true
+```
+
+`shm_size: 512mb` is usually enough; `stdin_open` and `tty` are only needed if your orchestrator runs the container without a TTY.
+
+### Local troubleshooting
+
+1. Download [uv](https://docs.astral.sh/uv/getting-started/installation/)
+2. Download dependencies using `uv sync --group test`
+3. Run tests with `uv run pytest --retries 3` (You can add `-n auto` for parallelization)
+4. If you see any `F` character in terminal, that means test failed even after retries.
+5. Depending of the test success:
+   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
+   2. If test fails, try troubleshooting on another host/using other method

compose.yaml

@@ -2,6 +2,8 @@ services:
   byparr:
     image: ghcr.io/thephaseless/byparr:latest
     restart: unless-stopped
+    # environment:
+    #   LOG_LEVEL: debug
     build:
       context: .
       dockerfile: Dockerfile

icon/logo-byparr.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="M243.9 511.9c-58.1-3.2-110.4-24-155-61.8-7-5.9-21.2-20.1-27.2-27.2-20-23.7-35.2-49.2-45.4-76.5-8.3-22-13-42.6-15.5-67.3-1.1-10.8-1.1-35.7 0-46.5 5.6-54.9 25.6-101.9 61-143.8 5.9-7 20.1-21.2 27.2-27.2 23.7-20 49.2-35.2 76.5-45.4 22-8.3 42.7-13 67.3-15.5 10.8-1.1 35.7-1.1 46.5 0 29.1 3 55.1 9.7 80 20.7 39.2 17.3 72.8 43.6 99.6 78 5.3 6.7 14.8 21 19.4 29.1 15.8 27.6 26.4 58.3 31.1 89.8 2 13.6 2.5 21.5 2.5 37.5s-.5 23.9-2.5 37.5c-7.1 47.8-27.1 91.6-59.2 129.5-5.9 7-20.1 21.2-27.2 27.2-16.4 13.8-34.2 25.8-51.8 34.6-28.8 14.5-59.1 23.1-91.7 26.3-7.4.8-28.9 1.3-35.6 1" style="fill:#fff"/><path d="M246.5 395.1c-1.7-.2-3.9-.3-5-.2-1 .1-3.5 0-5.5-.2-2-.3-6.1-.8-9.3-1.2s-6.7-.9-7.8-1.2c-1.2-.3-7.7-3.2-14.5-6.6s-14.3-7-16.8-8.1c-8.5-3.7-14.5-8.1-18.1-13.4-2.1-3.1-3.7-4.4-8.2-6.7-4-2-6-4.1-10.9-11-2-2.9-5-6.8-6.6-8.7-10.9-12.9-11.1-13.3-12.4-19.5-.5-2.5-1.6-5.4-2.9-7.8-4-7.4-5.4-10.8-6.4-14.9-.5-2.3-1.7-6.6-2.6-9.6-1.5-5.1-1.7-6.1-2.1-15.8-.4-9.4-.4-10.9.5-16.1.5-3.2 1.1-7.4 1.2-9.4.2-2 .6-5.4 1-7.6s.9-6.5 1.2-9.5c1-9.8 2.6-14.1 9.1-23.8 4.2-6.2 5.2-8.5 6-13.5.6-3.9 1.4-5.7 4.1-8.9 1.2-1.5 3.5-5.4 5.1-8.6 3.1-6.4 3.2-6.5 8.2-9.9 1.9-1.3 5.8-4 8.6-6s6.3-4.4 7.6-5.4c1.7-1.2 3.1-2.7 4.3-4.6 4.1-6.6 6.1-8 16.3-11.7 3.1-1.1 7.5-3.1 9.8-4.5 5.3-3.1 9-4.4 15-5 2.8-.3 6.8-1.2 9.8-2.1 4.5-1.4 5.6-1.6 10.4-1.6 5.7 0 6.4-.2 13.2-3.1 4.8-2.1 8.9-2.5 14.8-1.5 2.7.5 7.2 1.1 10.1 1.5 6.2.8 6.1.8 11.2 3.1 3.4 1.5 4.9 1.9 9.3 2.3 7 .7 7.9 1 16.1 5.1 3.9 2 9.5 4.5 12.4 5.7 9.8 3.9 14.3 7.2 22.1 15.7 3.3 3.6 8.6 9.2 11.9 12.5 14.5 14.5 14.9 15 16.5 19.4.8 2.2 2.6 6 3.9 8.3 5.2 9.2 12.2 23.8 13.4 28 .7 2.3 1.9 6.5 2.7 9.3 2.1 6.9 2.4 14.2 1 23.6-1.2 8-1.2 11.9-.3 18 .6 3.6.6 5.5.3 9.4-.7 6.7-1.6 10-4.1 14.1-1.3 2.2-2.4 4.7-2.9 6.6-1.8 7.8-5.6 16.3-10.7 24.3-1.4 2.2-3.4 5.5-4.3 7.4-2.2 4.3-4.2 6.6-8.2 9.6-3.7 2.7-14.5 14-17.7 18.4-2.2 3.1-7.1 7.4-10.6 9.4-1.1.6-4.3 1.8-7.2 2.6s-7.1 2.5-9.5 3.7c-3.6 1.8-5 3-8.6 6.6-2.3 2.4-4.9 4.7-5.7 5.1-1.6.8-5.5 1.5-10.7 2-1.7.2-4.6.8-6.4 1.5-7 2.6-7.3 2.6-16.1 2.7-6.9.1-9.3.3-12.3 1.2-4.9.9-8.8 1.2-12.7.6" style="fill:#cc7a3d"/><path d="M237.7 473.6c-42-3.8-79.9-18.3-113-43.2C56 378.7 24.1 291.7 43.1 207.6c6.6-29.2 19.8-57.8 37.7-81.9 47.7-64.1 126.8-97.1 205.8-85.9 36 5.1 70.5 19.3 99.7 41C455.7 132.5 488 220 468.9 304.4c-11.4 50.4-40.9 95.7-82.6 126.8-29.1 21.7-63.7 35.9-99.7 41-15.1 2.2-34.5 2.7-48.9 1.4m33.7-5.2c29-2.5 54.4-9.5 79-21.8 41.4-20.6 75.5-54.8 96.1-96.1 16.9-34 24.9-74.2 21.8-109.9-3.7-42.4-17.6-79.2-42.3-112-34.4-45.8-84.7-75.5-141.4-83.5-25.2-3.6-49.5-2.5-75.2 3.3-49.6 11.2-93.7 40-124.4 81.2-21.3 28.7-35 62-40.1 97.8-2.8 19.4-2.8 38 0 57.4 8.1 56.7 37.7 107 83.5 141.4 11.1 8.3 21 14.5 33 20.5 34.1 16.9 74.4 24.8 110 21.7" style="fill:#b1b1b1"/><path d="M253.9 367.3c-2.9-.7-4.3-1.4-6-2.9-2.9-2.6-3.9-4.9-3.9-8.9 0-3.9 1-6.3 3.9-8.9 6-5.4 15.8-3.1 19 4.3 1.1 2.6 1 7-.3 9.6-1.2 2.4-3.7 4.9-6 5.8-1.7.8-5.3 1.3-6.7 1m68.4-29.3c-9.2-3.4-11-15.4-3.1-21.1 8.1-5.9 20 1 18.8 11-.5 4-3 7.6-6.7 9.5-2.4 1.2-6.6 1.5-9 .6m-139.8-.2c-3.4-1.1-6.2-3.5-7.9-6.7-1.3-2.4-1.2-7.7.2-10.2 1.2-2.3 3.9-4.8 6.3-5.8 2.6-1.1 7-1 9.6.3 4.2 2.1 6.7 6 6.7 10.7 0 6.5-4.7 11.5-11 11.9-1.5.1-3.3 0-3.9-.2m43.2-22.8c-2.4-.8-4.1-1.9-5.8-3.8-5.4-6-3.1-15.8 4.3-19 2.6-1.1 7-1 9.6.3 2.5 1.2 4.9 3.7 5.9 6.2.9 2.1 1 6.3.1 8.8-.8 2.3-3.5 5.3-5.9 6.6-1.9 1-6.3 1.5-8.2.9m54.6 0c-2.4-.8-4.1-1.9-5.8-3.8-5.4-6-3.1-15.8 4.3-19 2.6-1.1 7-1 9.6.3 2.5 1.2 4.9 3.7 5.9 6.2.9 2.1 1 6.3.1 8.8-.8 2.3-3.5 5.3-5.9 6.6-1.9 1-6.3 1.5-8.2.9m-27.6-47c-2.2-.7-4-1.9-5.5-3.5-2.5-2.7-3.2-4.5-3.2-8.5 0-3.9.7-5.8 3.2-8.5s4.8-3.6 8.8-3.6 6.3.9 8.8 3.6 3.2 4.5 3.2 8.5c0 2.5-.3 4-.9 5.2-1.2 2.3-3.5 4.7-5.7 5.8-1.9 1.1-6.7 1.6-8.7 1m99.7 0c-2.4-.8-4.1-1.9-5.8-3.8-5.4-6-3.1-15.8 4.3-19 2.6-1.1 7-1 9.6.3 2.5 1.2 4.9 3.7 5.9 6.2.9 2.1 1 6.3.1 8.8-.8 2.3-3.5 5.3-5.9 6.6-1.9 1-6.3 1.5-8.2.9m-198.9-.7c-8.9-2.5-11.8-13.8-5.1-20.1 2.6-2.5 4.5-3.2 8.2-3.2 3.9 0 6.3 1 8.9 3.9 2.1 2.3 3 4.7 2.9 8.2 0 7.6-7.5 13.2-14.9 11.2m42.9-.4c-2.3-1.2-5.2-4.1-6.1-6.4-1-2.4-.9-7.1.3-9.5 1.1-2.3 4.1-5.2 6.4-6.2 2.4-1 7.1-.9 9.5.3 2.3 1.1 5.2 4.1 6.2 6.4 1 2.4.9 7.1-.3 9.5-1.1 2.3-4.1 5.2-6.4 6.2-2.5 1.1-7.2.9-9.6-.3m109-.3c-11.1-5.6-7-22.5 5.5-22.4 3.7 0 6.1 1.1 8.6 3.9 2.2 2.4 3.2 4.9 3.2 8s-1 5.6-3.2 8c-2.5 2.8-5 3.9-8.6 3.9-2.3-.1-3.5-.4-5.5-1.4m-79.7-46.3c-8.9-2.5-11.8-13.8-5.1-20.1 2.6-2.5 4.5-3.2 8.2-3.2 3.9 0 6.3 1 8.9 3.9 2.1 2.3 3 4.7 2.9 8.2-.1 7.6-7.6 13.3-14.9 11.2m54.5 0c-8-2.2-11.3-12-6.4-18.5 5.7-7.5 16.1-6.4 20.5 2 1.3 2.4 1.2 7.7-.2 10.2-2.6 5.1-8.6 7.8-13.9 6.3m43.2-22.8c-8-2.2-11.3-12-6.4-18.5 5.7-7.5 16.1-6.4 20.5 2 1.3 2.4 1.2 7.7-.2 10.2-2.6 5.2-8.6 7.8-13.9 6.3m-140.6-.5c-8.7-2.4-11.6-13-5.4-19.7 2.5-2.6 4.8-3.6 8.6-3.6 3.9 0 6.3 1.1 8.8 3.8 2.2 2.5 3 4.7 3 8 .1 8-7.3 13.6-15 11.5m70.3-28.9c-8-2.2-11.3-12-6.4-18.5 5.7-7.5 16.1-6.4 20.5 2 1.3 2.4 1.2 7.7-.2 10.2-2.6 5.1-8.5 7.8-13.9 6.3" style="fill:#884717"/></svg>

main.py

@@ -2,30 +2,32 @@ from __future__ import annotations
 
 import asyncio
 import logging
-import os
 import sys
 
 import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
 
-from src.consts import LOG_LEVEL, VERSION
+from src.consts import HOST, LOG_LEVEL, PORT, VERSION
 from src.endpoints import health_check, router
 from src.middlewares import LogRequest
-from src.utils import get_camoufox, logger
+from src.owui import router as owui_router
+from src.utils import get_browser, logger
 
 logger.info("Using version %s", VERSION)
+logger.info("Log level set to %s", logging.getLevelName(LOG_LEVEL))
 
 app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL)
 app.add_middleware(GZipMiddleware)
 app.add_middleware(LogRequest)
 
 app.include_router(router=router)
+app.include_router(router=owui_router)
 
 
 async def init():
     """Initialize the application."""
-    async for browser in get_camoufox():
+    async for browser in get_browser():
         await health_check(browser)
 
 
@@ -38,5 +40,4 @@ if __name__ == "__main__":
         loop.run_until_complete(init())
         logger.info("Initialization complete.")
     else:
-        host = os.getenv("HOST", "0.0.0.0")  # noqa: S104
+        uvicorn.run(app, host=HOST, port=PORT)
-        uvicorn.run(app, host=host, port=8191)

pyproject.toml

@@ -2,34 +2,34 @@
 [project]
 authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
 license = { text = "LICENSE" }
-requires-python = "<4.0,>=3.12"
+requires-python = "==3.14.*"
 name = "Byparr"
 version = "0.1.0"
 description = "API for getting cookies for Cloudflare challenges"
 readme = "README.md"
 dependencies = [
-    "camoufox[geoip]==0.4.*",
+    "fastapi[standard]==0.141.*",
-    "fastapi[standard]==0.116.*",
+    "invisible-playwright>=0.6.1",
+    "playwright==1.63.*",
     "playwright-captcha==0.1.*",
-    "pydantic==2.11.*",
+    "pydantic==2.*",
+    "pydantic-settings==2.*",
+    "trafilatura==2.2.*",
 ]
 urls = { repository = "https://github.com/ThePhaseless/Byparr" }
-
 [dependency-groups]
+
 test = [
-    "httpx==0.28.*",
+    "httpx2==2.13.*",
-    "pytest==8.4.*",
+    "pytest==9.1.*",
-    "pytest-asyncio==1.1.*",
+    "pytest-asyncio==1.4.*",
-    "pytest-progress==1.3.*",
     "pytest-retry==1.7.*",
     "pytest-xdist==3.8.*",
     "setuptools>=80.9.0",
 ]
-dev = ["deptry==0.23.*", "ruff==0.12.*"]
+dev = ["deptry==0.25.*", "ruff==0.16.*"]
-
 [tool.deptry.per_rule_ignores]
 DEP002 = ["pyautogui"]
-
 [tool.ruff.lint]
 ignore = [
     "D203",
@@ -54,7 +54,8 @@ ignore = [
     "G004",
     "ANN001",
     "ANN204",
-    "ANN206",
+    "CPY001",
+    "BLE001",
 ]
 select = ["ALL"]
 extend-safe-fixes = ["D415"]

renovate.json

@@ -2,6 +2,16 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:recommended"],
   "forkProcessing": "enabled",
+  "lockFileMaintenance": {
+    "enabled": true,
+    "rebaseWhen": "behind-base-branch",
+    "branchTopic": "lock-file-maintenance",
+    "commitMessageAction": "Lock file maintenance",
+    "schedule": ["before 4am on monday"],
+    "prBodyDefinitions": {
+      "Change": "All locks refreshed"
+    }
+  },
   "packageRules": [
     {
       "automerge": true,

src/challenge.py

@@ -0,0 +1,117 @@
+import time
+from asyncio import sleep
+from contextlib import suppress
+
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import FloatRect, Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from playwright_captcha.solvers.click.cloudflare.utils.detection import (
+    CF_INTERSTITIAL_INDICATORS_SELECTORS,
+    detect_cloudflare_challenge,
+)
+
+from src.utils import TimeoutTimer, logger
+
+__all__ = [
+    "CF_INTERSTITIAL_INDICATORS_SELECTORS",
+    "challenge_present",
+    "solve_challenge",
+]
+
+POLL_INTERVAL = 0.25
+CLICK_SETTLE = 1.5
+CLICK_COOLDOWN = 4
+PROBE_INTERVAL = 0.5
+TOKEN_READ_TIMEOUT = 1000
+BOX_READ_TIMEOUT = 1000
+CHECKBOX_INSET = 25
+TURNSTILE_INPUT = 'input[name="cf-turnstile-response"]'
+WIDGET_ANCESTOR_DEPTHS = (1, 2, 3, 4)
+WIDGET_MIN_WIDTH = 40
+WIDGET_MIN_HEIGHT = 20
+WIDGET_MAX_HEIGHT = 120
+
+
+async def challenge_present(page: Page) -> bool:
+    """Report whether the Cloudflare interstitial is up."""
+    return await detect_cloudflare_challenge(page, "interstitial")
+
+
+async def widget_box(page: Page) -> FloatRect | None:
+    """Measure the widget container with locators; running page scripts resets the challenge."""
+    for depth in WIDGET_ANCESTOR_DEPTHS:
+        widget = page.locator(f"{TURNSTILE_INPUT} >> xpath=ancestor::div[{depth}]")
+        with suppress(PlaywrightError, PlaywrightTimeoutError):
+            if await widget.count() == 0:
+                continue
+            box = await widget.first.bounding_box(timeout=BOX_READ_TIMEOUT)
+            if (
+                box
+                and box["width"] > WIDGET_MIN_WIDTH
+                and WIDGET_MIN_HEIGHT < box["height"] < WIDGET_MAX_HEIGHT
+            ):
+                return box
+    return None
+
+
+async def click_checkbox(page: Page) -> bool:
+    """Click the checkbox through its container, leaving its closed shadow root alone."""
+    box = await widget_box(page)
+    if box is None:
+        return False
+    await page.mouse.move(box["x"] + CHECKBOX_INSET, box["y"] + box["height"] / 2)
+    try:
+        await page.mouse.down()
+    finally:
+        await page.mouse.up()
+    return True
+
+
+async def checkbox_already_answered(page: Page) -> bool:
+    """Report whether Turnstile has already filled in its response token."""
+    token = page.locator(TURNSTILE_INPUT)
+    with suppress(PlaywrightError, PlaywrightTimeoutError):
+        if await token.count() > 0:
+            return bool(await token.first.input_value(timeout=TOKEN_READ_TIMEOUT))
+    return False
+
+
+async def challenge_is_gone(page: Page) -> bool:
+    """Confirm the interstitial is really gone and not just between navigations."""
+    if await challenge_present(page):
+        return False
+    await sleep(POLL_INTERVAL)
+    return not await challenge_present(page)
+
+
+async def solve_challenge(page: Page, timer: TimeoutTimer) -> None:
+    """Wait out the interstitial, clicking its checkbox whenever one is offered."""
+    logger.info("Challenge detected, waiting for it to clear...")
+    clicks = 0
+    next_click = 0.0
+    while True:
+        if await challenge_is_gone(page):
+            logger.debug("Challenge cleared.")
+            if clicks:
+                await sleep(min(CLICK_SETTLE, timer.remaining()))
+            return
+
+        if time.perf_counter() >= next_click:
+            landed = False
+            with suppress(PlaywrightError, PlaywrightTimeoutError):
+                landed = not await checkbox_already_answered(
+                    page
+                ) and await click_checkbox(page)
+            if landed:
+                clicks += 1
+                logger.info("Clicked the challenge checkbox (attempt %d).", clicks)
+            next_click = time.perf_counter() + (
+                CLICK_COOLDOWN if landed else PROBE_INTERVAL
+            )
+
+        if timer.remaining() <= 0:
+            break
+        await sleep(POLL_INTERVAL)
+
+    message = "Challenge still present when the request budget ran out"
+    raise TimeoutError(message)

src/consts.py

@@ -1,29 +1,41 @@
 import logging
-import os
-from pathlib import Path
 
-from playwright_captcha import CaptchaType
+from pydantic_settings import BaseSettings, SettingsConfigDict
-from playwright_captcha.utils.camoufox_add_init_script.add_init_script import (
-    get_addon_path,
-)
-
-LOG_LEVEL = logging.getLevelNamesMapping()[os.getenv("LOG_LEVEL", "INFO").upper()]
-
-VERSION = os.getenv("VERSION", "unknown").removeprefix("v")
-
-ADDON_PATH = str(Path(get_addon_path()).absolute())
-MAX_ATTEMPTS = 2**10
 
 
-PROXY_SERVER = os.getenv("PROXY_SERVER")
+class Settings(BaseSettings):
-PROXY_USERNAME = os.getenv("PROXY_USERNAME")
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
-PROXY_PASSWORD = os.getenv("PROXY_PASSWORD")
 
-CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = {
+    log_level: str = "INFO"
-    # Cloudflare
+    version: str = "unknown"
-    CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."],
-}
 
-CHALLENGE_TITLES = [
+    proxy_server: str | None = None
-    title for titles in CHALLENGE_TITLES_MAP.values() for title in titles
+    proxy_username: str | None = None
-]
+    proxy_password: str | None = None
+
+    host: str = "0.0.0.0"  # noqa: S104
+    port: int = 8191
+
+    block_media: bool = False
+    return_only_cookies: bool = False
+    owui_api_key: str | None = None
+    browser_locale: str | None = None
+
+
+settings = Settings()
+
+LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()]
+VERSION = settings.version.removeprefix("v")
+
+PROXY_SERVER = settings.proxy_server
+PROXY_USERNAME = settings.proxy_username
+PROXY_PASSWORD = settings.proxy_password
+
+HOST = settings.host
+PORT = settings.port
+
+BLOCK_MEDIA = settings.block_media
+RETURN_ONLY_COOKIES = settings.return_only_cookies
+
+OWUI_API_KEY = settings.owui_api_key
+BROWSER_LOCALE = settings.browser_locale

src/content.py

@@ -0,0 +1,42 @@
+import base64
+
+from playwright.async_api import Page
+
+from src.models import LinkRequest
+from src.utils import logger
+
+
+async def build_response_content(
+    page: Page,
+    request: LinkRequest,
+    page_request: object,
+    *,
+    challenge_detected: bool,
+    page_html: str | None,
+) -> tuple[str, str]:
+    """Build (content_type, response_content) from the settled page."""
+    if request.return_only_cookies:
+        return "text/html", ""
+
+    if page_request and page_request.headers.get("content-type", "").startswith(
+        "application/pdf"
+    ):
+        return await fetch_pdf_content(page)
+
+    response_content = (
+        page_html
+        if page_html is not None and not challenge_detected
+        else await page.content()
+    )
+    return "text/html", response_content
+
+
+async def fetch_pdf_content(page: Page) -> tuple[str, str]:
+    """Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
+    try:
+        fetch_response = await page.request.fetch(page.url)
+        response_content = base64.b64encode(await fetch_response.body()).decode("ascii")
+    except Exception:
+        logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
+        return "text/html", await page.content()
+    return "application/pdf", response_content

src/endpoints.py

@@ -1,29 +1,35 @@
 import time
 import warnings
-from asyncio import wait_for
 from http import HTTPStatus
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
-from httpx import TimeoutException
+from playwright.async_api import Error as PlaywrightError
-from playwright_captcha import CaptchaType
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 
-from src.consts import CHALLENGE_TITLES
+from src.challenge import challenge_present, solve_challenge
+from src.content import build_response_content
 from src.models import (
     HealthcheckResponse,
     LinkRequest,
     LinkResponse,
     Solution,
 )
-from src.utils import CamoufoxDepType, get_camoufox, logger
+from src.utils import (
+    BrowserDepClass,
+    TimeoutTimer,
+    get_browser,
+    logger,
+    remaining_ms,
+)
 
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
 
 router = APIRouter()
 
-CamoufoxDep = Annotated[CamoufoxDepType, Depends(get_camoufox)]
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
 
 
 @router.get("/", include_in_schema=False)
@@ -34,7 +40,7 @@ def read_root():
 
 
 @router.get("/health")
-async def health_check(sb: CamoufoxDep):
+async def health_check(sb: BrowserDep):
     """Health check endpoint."""
     health_check_request = await read_item(
         LinkRequest.model_construct(url="https://google.com"),
@@ -51,48 +57,99 @@ async def health_check(sb: CamoufoxDep):
 
 
 @router.post("/v1")
-async def read_item(request: LinkRequest, dep: CamoufoxDep) -> LinkResponse:
+async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     """Handle POST requests."""
     start_time = int(time.time() * 1000)
-
+    timer = TimeoutTimer(duration=request.max_timeout)
     request.url = request.url.replace('"', "").strip()
-    page_request = await dep.page.goto(request.url)
-    await dep.page.wait_for_load_state(state="domcontentloaded")
-    await dep.page.wait_for_load_state("networkidle")
 
-    if await dep.page.title() in CHALLENGE_TITLES:
+    await setup_routes(request, dep)
-        logger.info("Challenge detected, attempting to solve...")
+
-        # Solve the captcha
+    try:
-        remaining_timeout = (
+        challenge_detected, page_html, page_request = await _navigate_and_solve(
-            request.max_timeout - (int(time.time()) * 1000 - start_time) / 1000
+            dep, request, timer
         )
-        logger.debug(
+    except (TimeoutError, PlaywrightTimeoutError) as e:
-            "Remaining timeout for solving the challenge: %d ms", remaining_timeout
+        logger.error("Timed out while loading the page or solving the challenge")
-        )
+        raise HTTPException(
-        try:
+            status_code=408,
-            await wait_for(
+            detail="Timed out while loading the page or solving the challenge",
-                dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
+        ) from e
-                    captcha_container=dep.page,
+    except PlaywrightError as e:
-                    captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
+        logger.error("Could not reach the target: %s", e)
-                ),
+        raise HTTPException(
-                timeout=remaining_timeout,
+            status_code=502,
-            )
+            detail=f"Could not reach the target: {e}",
-        except TimeoutException as e:
+        ) from e
-            logger.error("Failed to solve challenge: %s", e)
-            return LinkResponse.invalid(url=request.url)
-        logger.debug("Challenge solved successfully.")
 
     cookies = await dep.context.cookies()
+    content_type, response_content = await build_response_content(
+        dep.page,
+        request,
+        page_request,
+        challenge_detected=challenge_detected,
+        page_html=page_html,
+    )
+
+    user_agent = (
+        page_request.request.headers.get("user-agent") or "" if page_request else ""
+    )
 
     return LinkResponse(
         message="Success",
         solution=Solution(
-            user_agent=await dep.page.evaluate("navigator.userAgent"),
+            user_agent=user_agent,
             url=dep.page.url,
-            status=page_request.status if page_request else HTTPStatus.OK,
+            status=HTTPStatus.OK,
             cookies=cookies,
             headers=page_request.headers if page_request else {},
-            response=await dep.page.content(),
+            response=response_content,
+            content_type=content_type,
         ),
         start_timestamp=start_time,
     )
+
+
+async def setup_routes(request: LinkRequest, dep: BrowserDep) -> None:
+    """Install request routes for media blocking."""
+    if request.block_media:
+
+        async def block_media_route(route) -> None:
+            if route.request.resource_type in ("image", "media", "font"):
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await dep.page.route("**/*", block_media_route)
+
+
+async def _navigate_and_solve(
+    dep: BrowserDep,
+    request: LinkRequest,
+    timer: TimeoutTimer,
+) -> tuple[bool, str | None, object]:
+    """Navigate to the URL, then solve a challenge or wait for network idle."""
+    page_html: str | None = None
+    page_request = await dep.page.goto(request.url, timeout=remaining_ms(timer))
+    await dep.page.wait_for_load_state(
+        state="domcontentloaded", timeout=remaining_ms(timer)
+    )
+
+    if not await challenge_present(dep.page):
+        page_html = await dep.page.content()
+        await _wait_for_networkidle(dep, timer)
+        return False, page_html, page_request
+
+    await solve_challenge(dep.page, timer)
+    await _wait_for_networkidle(dep, timer)
+    return True, page_html, page_request
+
+
+async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:
+    """Wait for network idle, tolerating post-DOM-load stalls."""
+    try:
+        await dep.page.wait_for_load_state("networkidle", timeout=remaining_ms(timer))
+    except PlaywrightTimeoutError:
+        logger.info(
+            "networkidle timed out after domcontentloaded; continuing with loaded page"
+        )

src/middlewares.py

@@ -25,6 +25,6 @@ class LogRequest(BaseHTTPMiddleware):
         if response.status_code == HTTPStatus.OK:
             logger.info(f"Done {request_body.url} in {process_time:.2f}s")
         else:
-            logger.info(f"Failed {request_body.url} in {process_time:.2f}s")
+            logger.warning(f"Failed {request_body.url} in {process_time:.2f}s")
 
         return response

src/models.py

@@ -5,13 +5,17 @@ from http.client import INTERNAL_SERVER_ERROR
 from typing import Any
 
 from playwright.sync_api import Cookie
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic.alias_generators import to_camel
 
 from src import consts
 
+MS_PER_SECOND = 1000
+
 
 class LinkRequest(BaseModel):
+    model_config = {"populate_by_name": True}
+
     cmd: str = Field(
         default="request.get",
         description="Type of request, currently only supports GET requests. This string is purely for compatibility with FlareSolverr.",
@@ -19,13 +23,36 @@ class LinkRequest(BaseModel):
     url: str = Field(pattern=r"^https?://", default="https://")
     max_timeout: int = Field(
         default=60,
-        description="Maximum timeout in seconds for resolving the anti-bot challenge.",
+        alias="maxTimeout",
+        description=(
+            "Maximum timeout for resolving the anti-bot challenge. Values below 1000 "
+            "are treated as seconds; values of 1000 or more as milliseconds, matching "
+            "FlareSolverr's maxTimeout parameter."
+        ),
     )
+    block_media: bool = Field(
+        default=consts.BLOCK_MEDIA,
+        alias="blockMedia",
+        description="Block image, media, and font resources from loading.",
+    )
+    return_only_cookies: bool = Field(
+        default=consts.RETURN_ONLY_COOKIES,
+        alias="returnOnlyCookies",
+        description="Return only cookies, skip the page HTML content in the response.",
+    )
+
+    @field_validator("max_timeout")
+    @classmethod
+    def normalize_max_timeout(cls, value: int) -> int:
+        """Normalize FlareSolverr-style millisecond values to seconds."""
+        if value >= MS_PER_SECOND:
+            return value // MS_PER_SECOND
+        return value
 
 
 class HealthcheckResponse(BaseModel):
     model_config = {"alias_generator": to_camel, "populate_by_name": True}
-    msg: str = "Byparr is ready!"
+    msg: str = "Byparr is working!"
     version: str = consts.VERSION
     user_agent: str
 
@@ -38,6 +65,7 @@ class Solution(BaseModel):
     user_agent: str = ""
     headers: dict[str, Any] = {}
     response: str = ""
+    content_type: str = Field(default="text/html", alias="contentType")
 
 
 class LinkResponse(BaseModel):
@@ -50,7 +78,7 @@ class LinkResponse(BaseModel):
     version: str = consts.VERSION
 
     @classmethod
-    def invalid(cls, url: str):
+    def invalid(cls, url: str) -> LinkResponse:
         """
         Return an invalid LinkResponse with default error values.
 

src/owui.py

@@ -0,0 +1,76 @@
+"""Open WebUI external web loader endpoint: POST /load."""
+
+from __future__ import annotations
+
+from hmac import compare_digest
+from typing import Annotated
+
+import trafilatura
+from fastapi import APIRouter, Depends, Header, HTTPException
+from playwright.async_api import Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from pydantic import BaseModel
+
+from src.consts import OWUI_API_KEY
+from src.utils import BrowserDepClass, get_browser, logger
+
+router = APIRouter(tags=["Open WebUI"])
+
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+
+class LoadRequest(BaseModel):
+    urls: list[str]
+
+
+class LoadResult(BaseModel):
+    page_content: str
+    metadata: dict[str, str]
+
+
+def require_auth(authorization: Annotated[str | None, Header()] = None) -> None:
+    """Enforce a bearer token on /load when OWUI_API_KEY is set."""
+    if not OWUI_API_KEY:
+        return
+    if authorization is None or not compare_digest(
+        authorization.encode(), f"Bearer {OWUI_API_KEY}".encode()
+    ):
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+async def _extract_content(page: Page) -> str:
+    """Return the page's main article text, falling back to visible text."""
+    article = trafilatura.extract(await page.content())
+    if article:
+        return article
+    result = await page.locator("body").inner_text()
+    return "\n".join(line.strip() for line in result.splitlines() if line.strip())
+
+
+@router.post("/load", response_model=list[LoadResult])
+async def load_urls(
+    request: LoadRequest,
+    _auth: Annotated[None, Depends(require_auth)],
+    dep: BrowserDep,
+) -> list[LoadResult]:
+    """
+    Fetch URLs through the anti-bot browser and return their text content.
+
+    Each URL is fetched sequentially; a failing URL yields empty
+    page_content so Open WebUI's RAG pipeline degrades gracefully.
+    """
+    results: list[LoadResult] = []
+    for url in request.urls:
+        try:
+            await dep.page.goto(url, timeout=60_000)
+            await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            try:
+                await dep.page.wait_for_load_state("networkidle", timeout=15_000)
+            except PlaywrightTimeoutError:
+                logger.debug("networkidle timed out for %s; extracting anyway", url)
+            content = await _extract_content(dep.page)
+        except Exception as exc:
+            logger.warning("Failed to load %s: %s", url, exc)
+            content = ""
+        results.append(LoadResult(page_content=content, metadata={"source": url}))
+    return results

src/utils.py

@@ -1,26 +1,28 @@
 import logging
+import time
 from collections.abc import AsyncGenerator
-from typing import NamedTuple, cast
+from typing import Annotated, NamedTuple, cast
 
-from camoufox import AsyncCamoufox
+from fastapi import Header
+from invisible_playwright.async_api import InvisiblePlaywright
 from playwright.async_api import Browser, BrowserContext, Page
-from playwright_captcha import (
+from pydantic import BaseModel, Field
-    ClickSolver,
-    FrameworkType,
-)
 
 from src.consts import (
-    ADDON_PATH,
+    BROWSER_LOCALE,
     LOG_LEVEL,
-    MAX_ATTEMPTS,
     PROXY_PASSWORD,
     PROXY_SERVER,
     PROXY_USERNAME,
 )
 
-solver_logger = logging.getLogger("playwright_captcha.solvers")
+solver_logger = logging.getLogger("playwright_captcha")
 solver_logger.handlers.clear()
-solver_logger.handlers.append(logging.NullHandler())
+if LOG_LEVEL == logging.DEBUG:
+    solver_logger.addHandler(logging.StreamHandler())
+    solver_logger.setLevel(LOG_LEVEL)
+else:
+    solver_logger.handlers.append(logging.NullHandler())
 
 logger = logging.getLogger("uvicorn.error")
 logger.setLevel(LOG_LEVEL)
@@ -28,44 +30,82 @@ if len(logger.handlers) == 0:
     logger.addHandler(logging.StreamHandler())
 
 
-class CamoufoxDepType(NamedTuple):
+class TimeoutTimer(BaseModel):
+    duration: int  # in seconds
+    start_time: float = Field(default_factory=time.perf_counter)
+
+    def remaining(self) -> float:
+        """Get remaining time in seconds."""
+        return max(0, self.duration - (time.perf_counter() - self.start_time))
+
+
+MIN_WAIT_MS = 1.0
+
+
+def remaining_ms(timer: TimeoutTimer) -> float:
+    """Milliseconds left, never 0 - Playwright reads that as no timeout at all."""
+    return max(MIN_WAIT_MS, timer.remaining() * 1000)
+
+
+class BrowserDepClass(NamedTuple):
     page: Page
-    solver: ClickSolver
     context: BrowserContext
 
 
-async def get_camoufox() -> AsyncGenerator[CamoufoxDepType, None]:
+async def get_browser(
-    """Get Camoufox instance."""
+    x_proxy_server: Annotated[
-    proxy_config = (
+        str | None,
-        {
+        Header(
+            alias="X-Proxy-Server",
+            description="Override proxy server for this request in protocol://host:port format.",
+        ),
+    ] = None,
+    x_proxy_username: Annotated[
+        str | None,
+        Header(
+            alias="X-Proxy-Username",
+        ),
+    ] = None,
+    x_proxy_password: Annotated[
+        str | None,
+        Header(
+            alias="X-Proxy-Password",
+        ),
+    ] = None,
+) -> AsyncGenerator[BrowserDepClass]:
+    """Get InvisiblePlaywright browser instance."""
+    header_server = x_proxy_server
+    header_username = x_proxy_username
+    header_password = x_proxy_password
+
+    proxy_config = None
+
+    if header_server:
+        proxy_config = {
+            "server": header_server,
+            "username": header_username,
+            "password": header_password,
+        }
+    elif PROXY_SERVER:
+        proxy_config = {
             "server": PROXY_SERVER,
             "username": PROXY_USERNAME,
             "password": PROXY_PASSWORD,
         }
-        if PROXY_SERVER
-        else None
-    )
 
-    async with AsyncCamoufox(
+    async with InvisiblePlaywright(
-        main_world_eval=True,
-        addons=[ADDON_PATH],
-        geoip=True,
-        proxy=proxy_config,
-        locale="en-US",
         headless=True,
+        proxy=proxy_config,
         humanize=True,
-        i_know_what_im_doing=True,
+        locale=BROWSER_LOCALE or "auto",
-        config={"forceScopeAccess": True},  # add this when creating Camoufox instance
+        extra_prefs={
-        disable_coop=True,  # add this when creating Camoufox instance
+            "devtools.jsonview.enabled": False,
+            "browser.tabs.remote.useCrossOriginOpenerPolicy": False,
+            "browser.tabs.remote.useCrossOriginEmbedderPolicy": False,
+        },
     ) as browser_raw:
-        # Cast to Browser since AsyncCamoufox always returns a Browser, not BrowserContext
+        # InvisiblePlaywright yields a Browser instance
         browser = cast("Browser", browser_raw)
         context = await browser.new_context()
         page = await context.new_page()
-        async with ClickSolver(
+        yield BrowserDepClass(page, context)
-            framework=FrameworkType.CAMOUFOX,
-            page=page,
-            max_attempts=MAX_ATTEMPTS,
-            attempt_delay=1,
-        ) as solver:
-            yield CamoufoxDepType(page, solver, context)

tests/main_test.py

@@ -1,12 +1,20 @@
+import base64
 from http import HTTPStatus
 from json import JSONDecodeError
+from unittest.mock import AsyncMock, MagicMock
 
-import httpx
+import httpx2
 import pytest
+from fastapi import HTTPException
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
+from src.challenge import CF_INTERSTITIAL_INDICATORS_SELECTORS
+from src.endpoints import read_item
 from src.models import LinkRequest
+from src.utils import BrowserDepClass, TimeoutTimer, remaining_ms
 
 client = TestClient(app)
 
@@ -27,7 +35,7 @@ def test_bypass(website: str):
 
     This test is skipped if the website is not reachable or does not have cloudflare/DDOS-GUARD.
     """
-    test_request = httpx.get(
+    test_request = httpx2.get(
         website,
     )
     if (
@@ -44,10 +52,44 @@ def test_bypass(website: str):
 
     response = client.post(
         "/v1",
-        json=LinkRequest.model_construct(url=website, cmd="request.get").model_dump(),
+        json=LinkRequest.model_construct(
+            url=website, cmd="request.get", max_timeout=60
+        ).model_dump(),
     )
 
     assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    assert "_cf_chl_opt" not in solution["response"]
+    assert "__cf_chl" not in solution["url"]
+
+
+def test_json_api():
+    """
+    JSON APIs must return 200, not crash on the UA evaluate.
+
+    Firefox renders application/json in a built-in viewer whose CSP blocks
+    Playwright's eval-based evaluate() (issue #394). The browser must be
+    launched with the viewer disabled so /v1 works and returns the raw JSON.
+    """
+    url = "https://api.ipify.org?format=json"
+    test_request = httpx2.get(url)
+    if test_request.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR:
+        pytest.skip(
+            f"Skipping JSON API test - upstream error ({test_request.status_code})"
+        )
+
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=url, cmd="request.get").model_dump(),
+    )
+
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping JSON API test - timed out (upstream issue)")
+
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    assert solution["userAgent"]
+    assert '"ip"' in solution["response"]
 
 
 def test_health_check():
@@ -59,3 +101,223 @@ def test_health_check():
     """
     response = client.get("/health")
     assert response.status_code == HTTPStatus.OK
+
+
+def test_pdf_handling():
+    """Tests that PDF URLs return the raw PDF bytes, not the Firefox viewer HTML."""
+    pdf_url = "https://mondaymandala.com/wp-content/uploads/Mickey-And-Minnie-Mouse-Holding-An-Easter-Egg-Basket-Coloring-Page-For-Kids.pdf"
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=pdf_url, cmd="request.get").model_dump(),
+    )
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping PDF test - timed out (upstream issue)")
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    if solution.get("contentType") != "application/pdf":
+        pytest.skip(
+            "Skipping PDF test - PDF bytes could not be fetched (upstream issue)"
+        )
+    assert solution["response"]  # non-empty base64
+
+    decoded = base64.b64decode(solution["response"])
+    assert decoded[:5] == b"%PDF-"
+
+
+@pytest.mark.parametrize(
+    ("payload", "expected"),
+    [
+        ({"max_timeout": 60}, 60),  # native API: seconds
+        ({"maxTimeout": 60}, 60),  # FlareSolverr alias, seconds-range value
+        ({"maxTimeout": 60000}, 60),  # FlareSolverr alias: milliseconds
+        ({"maxTimeout": 55000}, 55),
+        ({"maxTimeout": 1000}, 1),
+        ({}, 60),  # default
+    ],
+)
+def test_max_timeout_normalization(payload: dict, expected: int):
+    """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds."""
+    request = LinkRequest(url="https://example.com", **payload)
+    assert request.max_timeout == expected
+
+
+def fake_dep(
+    *,
+    fail_states: set[str] | None = None,
+    challenged: bool = False,
+    marker_counts: list[int] | None = None,
+    widget_box: dict[str, float] | None = None,
+    user_agent: str | None = "UnitTestBrowser/1.0",
+) -> BrowserDepClass:
+    """Build a browser dependency pair backed by mocks."""
+    page = AsyncMock()
+    page.url = "https://example.test/login"
+    page.goto.return_value = MagicMock(
+        status=HTTPStatus.OK,
+        headers={"content-type": "text/html"},
+        request=MagicMock(headers={"user-agent": user_agent} if user_agent else {}),
+    )
+    page.title.return_value = "Login"
+    page.content.return_value = "<html><title>Login</title></html>"
+    remaining = list(marker_counts or [])
+
+    def count_for(selector: str) -> int:
+        """Answer the marker check from the script, else from `challenged`."""
+        if selector not in CF_INTERSTITIAL_INDICATORS_SELECTORS or not remaining:
+            return 1 if challenged else 0
+        return remaining.pop(0) if len(remaining) > 1 else remaining[0]
+
+    def locator(selector: str) -> MagicMock:
+        handle = MagicMock()
+        handle.count = AsyncMock(side_effect=lambda: count_for(selector))
+        handle.first.bounding_box = AsyncMock(return_value=widget_box)
+        handle.first.input_value = AsyncMock(return_value="")
+        return handle
+
+    page.locator = MagicMock(side_effect=locator)
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        """Fail the wait when asked for a configured state."""
+        if state in (fail_states or set()):
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+
+    context = AsyncMock()
+    context.cookies.return_value = []
+    return BrowserDepClass(page=page, context=context)
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_after_domcontentloaded_returns_content():
+    """Pages that never go idle after DOM load must still return their content."""
+    dep = fake_dep(fail_states={"networkidle"})
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        dep,
+    )
+
+    assert response.status == "ok"
+    assert response.solution.response == "<html><title>Login</title></html>"
+
+
+@pytest.mark.asyncio
+async def test_domcontentloaded_timeout_returns_408():
+    """Fatal timeouts during initial page load still return a controlled 408."""
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login"),
+            fake_dep(fail_states={"domcontentloaded"}),
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT
+
+
+@pytest.mark.asyncio
+async def test_unreachable_host_is_a_502_not_a_500():
+    """An upstream we cannot reach is a gateway failure, never a Byparr crash."""
+    dep = fake_dep()
+    dep.page.goto.side_effect = PlaywrightError("Page.goto: NS_ERROR_UNKNOWN_HOST")
+
+    with pytest.raises(HTTPException) as exc:
+        await read_item(LinkRequest(url="https://nope.invalid/"), dep)
+
+    assert exc.value.status_code == HTTPStatus.BAD_GATEWAY
+
+
+@pytest.mark.asyncio
+async def test_status_is_always_ok_like_flaresolverr():
+    """FlareSolverr hardcodes 200 because Selenium cannot report the real code."""
+    dep = fake_dep()
+    dep.page.goto.return_value = MagicMock(
+        status=HTTPStatus.FORBIDDEN,
+        headers={"content-type": "text/html"},
+        request=MagicMock(headers={"user-agent": "UnitTestBrowser/1.0"}),
+    )
+
+    response = await read_item(LinkRequest(url="https://example.test/login"), dep)
+
+    assert response.solution.status == HTTPStatus.OK
+
+
+def test_exhausted_budget_never_disables_playwright_timeouts():
+    """Playwright reads timeout=0 as no timeout at all, so the floor must hold."""
+    spent = TimeoutTimer(duration=0)
+
+    assert spent.remaining() == 0
+    assert remaining_ms(spent) > 0
+
+
+@pytest.mark.asyncio
+async def test_missing_user_agent_header_is_not_a_500():
+    """A request without a user-agent header degrades to empty, never a 500 (#394)."""
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        fake_dep(user_agent=None),
+    )
+
+    assert response.status == "ok"
+    assert response.solution.user_agent == ""
+
+
+@pytest.mark.asyncio
+async def test_checkbox_is_clicked_while_the_challenge_is_up():
+    """A measurable widget gets a humanised press, not a raw synthetic click."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 1, 0],
+        widget_box={"x": 100.0, "y": 200.0, "width": 300.0, "height": 60.0},
+    )
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login", max_timeout=5), dep
+    )
+
+    assert response.status == "ok"
+    dep.page.mouse.move.assert_awaited_once_with(125.0, 230.0)
+    dep.page.mouse.down.assert_awaited_once()
+    dep.page.mouse.up.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_challenge_that_clears_on_its_own_is_never_clicked():
+    """A challenge is over when its markup goes, and until then we keep our hands off."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 0],
+        widget_box={"x": 100.0, "y": 200.0, "width": 300.0, "height": 60.0},
+    )
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login", max_timeout=5), dep
+    )
+
+    assert response.status == "ok"
+    dep.page.mouse.down.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_challenge_that_never_clears_returns_408():
+    """A challenge still up when the budget runs out is a timeout, not a 500."""
+    dep = fake_dep(challenged=True, marker_counts=[1])
+
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login", max_timeout=2), dep
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT
+
+
+@pytest.mark.asyncio
+async def test_marker_vanishing_mid_navigation_is_not_a_solved_challenge():
+    """The marker drops out between challenge rounds; one clear read proves nothing."""
+    dep = fake_dep(challenged=True, marker_counts=[1, 0, 1])
+
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login", max_timeout=2), dep
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT

tests/owui_test.py

@@ -0,0 +1,115 @@
+from http import HTTPStatus
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from starlette.testclient import TestClient
+
+from main import app
+from src.owui import LoadRequest, load_urls
+from src.utils import BrowserDepClass
+
+client = TestClient(app)
+
+
+def test_owui_load_basic():
+    """/load returns one result per URL with the expected shape."""
+    response = client.post("/load", json={"urls": ["https://example.com"]})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"]
+    assert results[0]["metadata"] == {"source": "https://example.com"}
+
+
+def test_owui_load_multiple_urls():
+    """/load returns one result per URL, in order."""
+    urls = ["https://example.com", "https://example.org"]
+    response = client.post("/load", json={"urls": urls})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert [r["metadata"]["source"] for r in results] == urls
+
+
+def test_owui_load_invalid_url_graceful():
+    """Unreachable URLs yield empty page_content instead of an error."""
+    response = client.post(
+        "/load", json={"urls": ["https://this-domain-does-not-exist-12345.invalid"]}
+    )
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"] == ""
+
+
+@pytest.mark.parametrize(
+    "headers",
+    [None, {"Authorization": "Bearer wrong-key"}],
+)
+def test_owui_load_rejects_missing_or_wrong_key(headers):
+    """/load returns 401 without a valid bearer token when a key is set."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load", json={"urls": ["https://example.com"]}, headers=headers
+        )
+        assert response.status_code == HTTPStatus.UNAUTHORIZED
+
+
+def test_owui_load_accepts_valid_key():
+    """/load succeeds with the configured bearer token."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load",
+            json={"urls": ["https://example.com"]},
+            headers={"Authorization": "Bearer test-secret-key"},
+        )
+        assert response.status_code == HTTPStatus.OK
+
+
+ARTICLE_HTML = """<html><head><title>Test</title></head><body>
+<article><h1>Example Title</h1><p>This is the main article body with enough words for
+trafilatura to consider it real content rather than boilerplate.</p></article>
+<nav><a href="/x">nav link</a></nav>
+</body></html>"""
+
+
+def fake_dep(*, html: str = ARTICLE_HTML) -> BrowserDepClass:
+    """Browser dependency whose page loads HTML but never reaches networkidle."""
+    page = AsyncMock()
+    page.goto.return_value = MagicMock()
+    page.content.return_value = html
+    page.locator = MagicMock()
+    page.locator.return_value.inner_text = AsyncMock(
+        return_value="line one\n\nline two"
+    )
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        if state == "networkidle":
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+    return BrowserDepClass(page=page, context=AsyncMock())
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_still_extracts_content():
+    """A page that never reaches networkidle still yields its article text."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]), None, fake_dep()
+    )
+    assert results[0].page_content == (
+        "Example TitleThis is the main article body with enough words for "
+        "trafilatura to consider it real content rather than boilerplate."
+    )
+
+
+@pytest.mark.asyncio
+async def test_extraction_falls_back_to_innertext():
+    """Pages trafilatura cannot score fall back to the rendered innerText."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]),
+        None,
+        fake_dep(html="<html><body></body></html>"),
+    )
+    assert results[0].page_content == "line one\nline two"