Merge pull request #384 from ThePhaseless/feat/owui-loader-cleanup

feat: add Open WebUI external web loader endpoint

.devcontainer/devcontainer.json

@@ -9,7 +9,11 @@
   "runArgs": ["--security-opt", "label=disable", "--privileged"],
   "customizations": {
     "vscode": {
-      "extensions": ["charliermarsh.ruff", "ms-python.python"],
+      "extensions": [
+        "charliermarsh.ruff",
+        "ms-python.python",
+        "tamasfe.even-better-toml"
+      ],
       "settings": {
         "python.venvPath": "./venv"
       }
@@ -17,7 +21,7 @@
   },
   "postCreateCommand": "uv sync --group test",
   "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+    "ghcr.io/devcontainers/features/docker-in-docker:4": {
       "moby": false
     }
   }

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -13,6 +13,8 @@ body:
         - label: Checked if such issue already exists
         - label: Checked other websites with Cloudflare Turnstile
         - label: I am able to access the webpage from my browser
+        - label: (Docker only) Checked that the problem does also occur on main tag
+
   - type: checkboxes
     attributes:
       label: "If checked other websites with Cloudflare Turnstile:"
@@ -20,11 +22,18 @@ body:
         - label: Other websites do work.
         - label: Other websites do not work.
 
+  - type: checkboxes
+    attributes:
+      label: "The issue is still present in the latest main tag:"
+      options:
+        - label: "Yes"
+        - label: "No"
+
   - type: input
     id: docker-host
     attributes:
       label: OS/Docker Host
-      placeholder: e.g. Docker, WSL2, Ubuntu, Windows 
+      placeholder: e.g. Docker, WSL2, Ubuntu, Windows
 
   - type: input
     id: platform-model
@@ -49,7 +58,7 @@ body:
       label: Describe the bug
       description: A clear and concise description of what the bug is.
       placeholder: "Example: The application crashes when trying to load a new file"
-  
+
   - type: textarea
     id: config
     attributes:

.github/workflows/docker-cleanup.yml

@@ -0,0 +1,107 @@
+name: Cleanup PR Docker Images
+
+on:
+  pull_request:
+    types: [closed]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Delete PR Docker images
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const owner = context.repo.owner.toLowerCase();
+            const repo = context.repo.repo.toLowerCase();
+            const prNumber = context.payload.pull_request.number;
+
+            // Get the package
+            const packageName = `${repo}`;
+
+            console.log(`Cleaning up images for PR #${prNumber}`);
+
+            // Check if owner is an org or user
+            let isOrg = false;
+            try {
+              await github.rest.orgs.get({ org: owner });
+              isOrg = true;
+              console.log(`Repository owner is an organization`);
+            } catch (error) {
+              console.log(`Repository owner is a user account`);
+            }
+
+            try {
+              // Get all versions of the package
+              let versions;
+              if (isOrg) {
+                versions = await github.rest.packages.getAllPackageVersionsForPackageOwnedByOrg({
+                  package_type: 'container',
+                  package_name: packageName,
+                  org: owner,
+                  per_page: 100
+                });
+              } else {
+                versions = await github.rest.packages.getAllPackageVersionsForPackageOwnedByUser({
+                  package_type: 'container',
+                  package_name: packageName,
+                  username: owner,
+                  per_page: 100
+                });
+              }
+
+              // Filter versions that match this PR (pr-123, pr-123-amd64, pr-123-arm64)
+              const prVersions = versions.data.filter(version => {
+                const tags = version.metadata?.container?.tags || [];
+                // Match tags like: pr-123, pr-123-amd64, pr-123-arm64
+                return tags.some(tag =>
+                  tag === `pr-${prNumber}` ||
+                  tag === `pr-${prNumber}-amd64` ||
+                  tag === `pr-${prNumber}-arm64`
+                );
+              });
+
+              console.log(`Found ${prVersions.length} image version(s) to delete`);
+
+              // Delete each version
+              for (const version of prVersions) {
+                console.log(`Deleting version ${version.id} with tags: ${version.metadata?.container?.tags?.join(', ')}`);
+                try {
+                  if (isOrg) {
+                    await github.rest.packages.deletePackageVersionForOrg({
+                      package_type: 'container',
+                      package_name: packageName,
+                      org: owner,
+                      package_version_id: version.id
+                    });
+                  } else {
+                    await github.rest.packages.deletePackageVersionForUser({
+                      package_type: 'container',
+                      package_name: packageName,
+                      username: owner,
+                      package_version_id: version.id
+                    });
+                  }
+                  console.log(`✓ Deleted version ${version.id}`);
+                } catch (error) {
+                  console.error(`Failed to delete version ${version.id}:`, error.message);
+                }
+              }
+
+              console.log('Cleanup completed');
+            } catch (error) {
+              if (error.status === 404) {
+                console.log('No package found - nothing to clean up');
+              } else {
+                console.error('Error during cleanup:', error.message);
+                throw error;
+              }
+            }

.github/workflows/docker-publish.yml

@@ -12,8 +12,26 @@ on:
     branches: ["*"]
     # Publish semver tags as releases.
     tags: ["v*.*.*"]
+    paths:
+      - "src/**"
+      - "main.py"
+      - "tests/**"
+      - "Dockerfile"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "compose.yaml"
+      - ".github/workflows/docker-publish.yml"
   pull_request:
     branches: ["main"]
+    paths:
+      - "src/**"
+      - "main.py"
+      - "tests/**"
+      - "Dockerfile"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "compose.yaml"
+      - ".github/workflows/docker-publish.yml"
   workflow_dispatch:
 
 env:
@@ -36,14 +54,14 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Test
         id: test
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64
@@ -54,6 +72,8 @@ jobs:
 
   build:
     needs: test
+    # Skip build for Renovate PRs
+    if: github.event.pull_request.user.login != 'renovate[bot]'
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -66,26 +86,31 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       - name: Prepare variables
         id: vars
         run: |
           SURFIX=$(echo ${{ matrix.platform }} | cut -d'/' -f2)
           echo "SURFIX=$SURFIX" >> $GITHUB_OUTPUT
-          # Generate a unique local tag for the image
-          echo "LOCAL_TAG=${{ github.sha }}-$SURFIX" >> $GITHUB_OUTPUT
+
+          # Use PR number for PRs, SHA for everything else
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "LOCAL_TAG=pr-${{ github.event.pull_request.number }}-$SURFIX" >> $GITHUB_OUTPUT
+          else
+            echo "LOCAL_TAG=${{ github.sha }}-$SURFIX" >> $GITHUB_OUTPUT
+          fi
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       # Set up BuildKit Docker container builder
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Log into registry
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -94,19 +119,19 @@ jobs:
       # Extract metadata (tags, labels) for Docker
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           tags: type=raw,value=${{ steps.vars.outputs.LOCAL_TAG }}
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      # Build and export Docker image for each platform (without pushing)
+      # Build and push Docker image for each platform
       - name: Build Docker image
         id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           pull: true
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ matrix.platform }}
@@ -119,7 +144,6 @@ jobs:
   merge-and-push:
     needs: build
     runs-on: ubuntu-latest
-    if: github.event_name != 'pull_request'
     permissions:
       contents: read
       packages: write
@@ -127,19 +151,19 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       # Install the cosign tool
       - name: Install cosign
-        uses: sigstore/cosign-installer@v3
+        uses: sigstore/cosign-installer@v4.1.2
 
       # Set up Docker Buildx
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       # Log into registry
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -148,7 +172,7 @@ jobs:
       # Extract Docker metadata for tagging
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           tags: |
             type=ref,event=branch
@@ -178,9 +202,16 @@ jobs:
 
           echo $args
 
-          docker buildx imagetools create $args \
-          ${image}:${{github.sha}}-amd64 \
-          ${image}:${{github.sha}}-arm64
+          # Use PR-based tags for PRs, SHA-based tags for everything else
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            docker buildx imagetools create $args \
+            ${image}:pr-${{ github.event.pull_request.number }}-amd64 \
+            ${image}:pr-${{ github.event.pull_request.number }}-arm64
+          else
+            docker buildx imagetools create $args \
+            ${image}:${{github.sha}}-amd64 \
+            ${image}:${{github.sha}}-arm64
+          fi
 
       # Sign the manifest
       - name: Sign the manifests

AGENTS.md

@@ -0,0 +1,37 @@
+# Copilot Instructions
+
+## Project overview
+
+- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using Camoufox + Playwright captcha solver.
+- Entry point: main app in main.py; routes and request flow defined in src/endpoints.py and src/models.py.
+- Browser lifecycle is owned by get_camoufox() in src/utils.py, which yields a page, solver, and context for each request.
+
+## Architecture and data flow
+
+- Request flow: POST /v1 -> read_item() -> browser.goto() -> wait for load states -> detect challenge title -> solve captcha -> return LinkResponse.
+- Challenge detection uses CHALLENGE_TITLES from src/consts.py; update this map when adding providers.
+- Health check hits /v1 internally with <https://google.com> and fails if status is not OK.
+- Logging: LogRequest middleware logs only POST /v1 timing and outcome; other paths pass through.
+
+## Key modules and patterns
+
+- Models use Pydantic v2 with camelCase aliasing for responses (see src/models.py).
+- LinkResponse.invalid() is the standard error response shape; keep fields consistent with FlareSolverr style.
+- get_camoufox() constructs AsyncCamoufox with addons and optional proxy config from env vars.
+
+## Config and environment
+
+- Core env vars in src/consts.py: HOST, PORT, PROXY_SERVER, PROXY_USERNAME, PROXY_PASSWORD, LOG_LEVEL, VERSION.
+- VERSION strips leading "v" for tag-style values.
+
+## Developer workflows
+
+- Local run: uv sync && uv run main.py
+- Init mode: uv run main.py --init (pre-warms health check via browser setup)
+- Tests: uv sync --group test && uv run pytest --retries 3
+- Docker troubleshooting: docker build --target test .
+
+## Tests and external dependencies
+
+- tests/main_test.py calls real websites; tests are network-dependent and may be skipped based on upstream status.
+- HTTP client tests use starlette.testclient + httpx; avoid mocking unless needed for local-only changes.

Dockerfile

@@ -1,5 +1,7 @@
-FROM debian:stable-slim AS base
-ENV HOME=/root
+# Ubuntu is required by playwright.
+# Pin to 24.04 LTS: ubuntu:latest floats to 26.04, which Playwright 1.58
+# cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch).
+FROM ubuntu:24.04 AS base
 
 ARG GITHUB_BUILD=false \
     VERSION
@@ -11,32 +13,51 @@ ENV GITHUB_BUILD=${GITHUB_BUILD}\
     # prevents python creating .pyc files
     PYTHONDONTWRITEBYTECODE=1 \
     UV_LINK_MODE=copy \
-    PATH="${HOME}/.local/bin:$PATH"
+    PORT=8191 \
+    XDG_CACHE_HOME=/cache \
+    HOME=/tmp
 
-WORKDIR /app
-
-RUN apt update && apt -y upgrade && apt install -y curl
-
-ADD https://astral.sh/uv/install.sh install.sh
-RUN sh install.sh && uv --version
-RUN uvx playwright install-deps firefox && uvx camoufox fetch
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 FROM base AS devcontainer
-RUN apt install -y git
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends git &&\
+    uvx playwright install-deps firefox &&\
+    uvx --from git+https://github.com/feder-cr/invisible_playwright.git python -m invisible_playwright fetch &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 ENTRYPOINT [ "sleep", "infinity" ]
 
-
 FROM base AS app
+WORKDIR /app
+
 COPY pyproject.toml uv.lock ./
-RUN --mount=type=cache,target=${HOME}/.cache/uv uv sync
+RUN mkdir -p /cache &&\
+    uv sync &&\
+    uv run python -m invisible_playwright fetch &&\
+    apt-get update &&\
+    uv run playwright install-deps firefox &&\
+    uv cache clean &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
+# Make app and cache world-readable; cache must be writable for runtime browser/profile data
+RUN chmod -R o+rX /app /cache &&\
+    chmod -R o+w /cache
+
 FROM app AS test
-RUN --mount=type=cache,target=${HOME}/.cache/uv uv sync --group test
-RUN uv run pytest --retries 3 -n auto
+RUN \
+    uv sync --group test &&\
+    uv run pytest --retries 3
 
 FROM app
-EXPOSE 8191
-HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
-ENTRYPOINT ["uv", "run", "main.py"]
+USER 1000
+EXPOSE $PORT
+HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health"
+ENTRYPOINT ["tini", "--", "/app/.venv/bin/python", "main.py"]

README.md

@@ -1,44 +1,22 @@
 # Byparr
 
-Built with [camoufox](https://camoufox.com/) and [FastAPI](https://fastapi.tiangolo.com), this project aims to mimic [FlareSolverr's](https://github.com/FlareSolverr/FlareSolverr) API and functionality of providing you with http cookies and headers for websites protected with anti-bot protections.
+<p align="center">
+   <img src="icon/logo-byparr.svg" alt="Byparr logo" width="120" />
+</p>
 
 > [!IMPORTANT]
 > This software does not **guarantee** (only greatly increases the chance) that any challenge will be bypassed. While this tool passes the initial browser check, Cloudflare and other captcha providers likely require valid network traffic originating from the user’s public IP address to mark a connection as legitimate. If any website does not pass the challenge, please run troubleshooting steps and check if other websites work before you create an GitHub issue.
 
-> [!IMPORTANT]
-> Support for NAS devices (like Synology) is minimal. Please report issues, but do not expect it to be fixed quickly. The only ARM device I have is a free Ampere Oracle VM, so I can only test ARM support on that. See [#22](https://github.com/ThePhaseless/Byparr/issues/22) and [#3](https://github.com/ThePhaseless/Byparr/issues/3)
-
-> [!NOTE]
-> Thanks to FastAPI implementation, now you can also see the API documentation at `/docs` or `/` (redirect to `/docs`) endpoints.
-
-## Troubleshooting
-
-### Docker
-
-1. Clone repo to the host that has issues with Byparr.
-2. Run `docker build --target test .`
-3. Depending of the build success:
-   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
-   2. If build fails, try troubleshooting on another host/using other method
-
-### Local
-
-1. Download [uv](https://docs.astral.sh/uv/getting-started/installation/)
-2. Download dependencies using `uv sync --group test`
-3. Run tests with `uv run pytest --retries 3` (You can add `-n auto` for parallelization)
-4. If you see any `F` character in terminal, that means test failed even after retries.
-5. Depending of the test success:
-   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
-   2. If test fails, try troubleshooting on another host/using other method
-
 ## Options
 
 | Environment Variable | Default   | Description                                                                                                                                                       |
 | -------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `HOST`               | `0.0.0.0` | Host address to bind the server to. Use `0.0.0.0` to bind to all IPv4 interfaces, `::` for all IPv6 interfaces, or `127.0.0.1`/`localhost` for local access only. |
+| `PORT`               | `8191`    | Port to bind the server to.                                                                                                                                       |
 | `PROXY_SERVER`       | None      | Proxy to use in format: `protocol://host:port`.                                                                                                                   |
-| `PROXY_USERNAME`     | None      |                                                                                                                                                                   |
-| `PROXY_PASSWORD`     | None      |                                                                                                                                                                   |
+| `PROXY_USERNAME`     | None      | Username for proxy authentication.                                                                                                                                |
+| `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
+| `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |
 
 ## Proxy Recommendation
 
@@ -48,30 +26,90 @@ Recently I've partnered with a _new in town_ proxy service - ProxyBase - to offe
 
 - `v*.*.*`/`latest` - Releases considered stable
 - `main` - Latest release from main branch (untested)
+- `pr-{number}` - Pull request images for testing (automatically cleaned up when PR closes)
 
 ## Usage
 
-### Docker Compose
+> [!IMPORTANT]
+> Support for NAS devices (like Synology) is minimal. Please report issues, but do not expect it to be fixed quickly. The only ARM device I have is a free Ampere Oracle VM, so I can only test ARM support on that. See [#22](https://github.com/ThePhaseless/Byparr/issues/22) and [#3](https://github.com/ThePhaseless/Byparr/issues/3)
 
-See `compose.yaml`
+### Docker Compose setup
 
-### Docker
+1. Review settings in `compose.yaml`.
+2. Start the service:
 
 ```bash
-docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
+docker compose up -d
 ```
 
-### Local
+### Docker install
+
+1. Pull and run the image:
+
+   ```bash
+   docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
+   ```
+
+2. Optional: set env vars using `-e` or `--env-file`.
+
+### Local install
+
+1. Install ([or update when Python version changes](https://github.com/astral-sh/uv/issues/17887)) [uv](https://docs.astral.sh/uv/getting-started/installation/).
+2. Clone this repo - `git clone https://github.com/ThePhaseless/Byparr`
+3. Run `uv run main.py`
+4. Enjoy!
+
+### API Docs
+
+Once running, open:
+
+- `http://localhost:8191/docs`
+- `http://localhost:8191/` (redirects to `/docs`)
+
+### Open WebUI Integration
+
+Byparr can serve as an external web loader for [Open WebUI](https://github.com/open-webui/open-webui), allowing it to fetch web content through Byparr's anti-bot bypassing capabilities.
+
+Configure Open WebUI with these environment variables:
 
 ```bash
-uv sync && uv run main.py
+WEB_LOADER_ENGINE=external
+EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load
+EXTERNAL_WEB_LOADER_API_KEY=your-secret-key  # Optional, must match OWUI_API_KEY
 ```
 
-## Need help with / TODO
+The `/load` endpoint accepts `POST` requests with `{"urls": ["https://..."]}` and returns extracted text content for RAG pipelines.
 
-- [x] Slimming container (only ~1.11 GB now!)
-- [x] Add more anti-bot challenges
-- [x] Add doc strings
-- [x] Implement versioning
-- [x] Proxy support
-- [x] Add more architectures support
+## Troubleshooting
+
+### Docker troubleshooting
+
+1. Clone repo to the host that has issues with Byparr.
+2. Run `docker build --target test .`
+3. Depending of the build success:
+   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
+   2. If build fails, try troubleshooting on another host/using other method
+
+#### Proxmox OCI / LXC browser launch errors
+
+If you are running Byparr as an OCI container in Proxmox (or another LXC-based setup) and see a `FileNotFoundError` from `multiprocessing.synchronize`/`camoufox` when processing requests, increase the service's shared memory in `compose.yaml`:
+
+```yaml
+services:
+  byparr:
+    shm_size: 512mb
+    stdin_open: true
+    tty: true
+```
+
+`shm_size: 512mb` is usually enough; `stdin_open` and `tty` are only needed if your orchestrator runs the container without a TTY.
+
+### Local troubleshooting
+
+1. Download [uv](https://docs.astral.sh/uv/getting-started/installation/)
+2. Download dependencies using `uv sync --group test`
+3. Run tests with `uv run pytest --retries 3` (You can add `-n auto` for parallelization)
+4. If you see any `F` character in terminal, that means test failed even after retries.
+5. Depending of the test success:
+   1. If run successfully, try updating container or if already on newest stable release create an issue for creating new release with new dependencies
+   2. If test fails, try troubleshooting on another host/using other method

icon/logo-byparr.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="M243.9 511.9c-58.1-3.2-110.4-24-155-61.8-7-5.9-21.2-20.1-27.2-27.2-20-23.7-35.2-49.2-45.4-76.5-8.3-22-13-42.6-15.5-67.3-1.1-10.8-1.1-35.7 0-46.5 5.6-54.9 25.6-101.9 61-143.8 5.9-7 20.1-21.2 27.2-27.2 23.7-20 49.2-35.2 76.5-45.4 22-8.3 42.7-13 67.3-15.5 10.8-1.1 35.7-1.1 46.5 0 29.1 3 55.1 9.7 80 20.7 39.2 17.3 72.8 43.6 99.6 78 5.3 6.7 14.8 21 19.4 29.1 15.8 27.6 26.4 58.3 31.1 89.8 2 13.6 2.5 21.5 2.5 37.5s-.5 23.9-2.5 37.5c-7.1 47.8-27.1 91.6-59.2 129.5-5.9 7-20.1 21.2-27.2 27.2-16.4 13.8-34.2 25.8-51.8 34.6-28.8 14.5-59.1 23.1-91.7 26.3-7.4.8-28.9 1.3-35.6 1" style="fill:#fff"/><path d="M246.5 395.1c-1.7-.2-3.9-.3-5-.2-1 .1-3.5 0-5.5-.2-2-.3-6.1-.8-9.3-1.2s-6.7-.9-7.8-1.2c-1.2-.3-7.7-3.2-14.5-6.6s-14.3-7-16.8-8.1c-8.5-3.7-14.5-8.1-18.1-13.4-2.1-3.1-3.7-4.4-8.2-6.7-4-2-6-4.1-10.9-11-2-2.9-5-6.8-6.6-8.7-10.9-12.9-11.1-13.3-12.4-19.5-.5-2.5-1.6-5.4-2.9-7.8-4-7.4-5.4-10.8-6.4-14.9-.5-2.3-1.7-6.6-2.6-9.6-1.5-5.1-1.7-6.1-2.1-15.8-.4-9.4-.4-10.9.5-16.1.5-3.2 1.1-7.4 1.2-9.4.2-2 .6-5.4 1-7.6s.9-6.5 1.2-9.5c1-9.8 2.6-14.1 9.1-23.8 4.2-6.2 5.2-8.5 6-13.5.6-3.9 1.4-5.7 4.1-8.9 1.2-1.5 3.5-5.4 5.1-8.6 3.1-6.4 3.2-6.5 8.2-9.9 1.9-1.3 5.8-4 8.6-6s6.3-4.4 7.6-5.4c1.7-1.2 3.1-2.7 4.3-4.6 4.1-6.6 6.1-8 16.3-11.7 3.1-1.1 7.5-3.1 9.8-4.5 5.3-3.1 9-4.4 15-5 2.8-.3 6.8-1.2 9.8-2.1 4.5-1.4 5.6-1.6 10.4-1.6 5.7 0 6.4-.2 13.2-3.1 4.8-2.1 8.9-2.5 14.8-1.5 2.7.5 7.2 1.1 10.1 1.5 6.2.8 6.1.8 11.2 3.1 3.4 1.5 4.9 1.9 9.3 2.3 7 .7 7.9 1 16.1 5.1 3.9 2 9.5 4.5 12.4 5.7 9.8 3.9 14.3 7.2 22.1 15.7 3.3 3.6 8.6 9.2 11.9 12.5 14.5 14.5 14.9 15 16.5 19.4.8 2.2 2.6 6 3.9 8.3 5.2 9.2 12.2 23.8 13.4 28 .7 2.3 1.9 6.5 2.7 9.3 2.1 6.9 2.4 14.2 1 23.6-1.2 8-1.2 11.9-.3 18 .6 3.6.6 5.5.3 9.4-.7 6.7-1.6 10-4.1 14.1-1.3 2.2-2.4 4.7-2.9 6.6-1.8 7.8-5.6 16.3-10.7 24.3-1.4 2.2-3.4 5.5-4.3 7.4-2.2 4.3-4.2 6.6-8.2 9.6-3.7 2.7-14.5 14-17.7 18.4-2.2 3.1-7.1 7.4-10.6 9.4-1.1.6-4.3 1.8-7.2 2.6s-7.1 2.5-9.5 3.7c-3.6 1.8-5 3-8.6 6.6-2.3 2.4-4.9 4.7-5.7 5.1-1.6.8-5.5 1.5-10.7 2-1.7.2-4.6.8-6.4 1.5-7 2.6-7.3 2.6-16.1 2.7-6.9.1-9.3.3-12.3 1.2-4.9.9-8.8 1.2-12.7.6" style="fill:#cc7a3d"/><path d="M237.7 473.6c-42-3.8-79.9-18.3-113-43.2C56 378.7 24.1 291.7 43.1 207.6c6.6-29.2 19.8-57.8 37.7-81.9 47.7-64.1 126.8-97.1 205.8-85.9 36 5.1 70.5 19.3 99.7 41C455.7 132.5 488 220 468.9 304.4c-11.4 50.4-40.9 95.7-82.6 126.8-29.1 21.7-63.7 35.9-99.7 41-15.1 2.2-34.5 2.7-48.9 1.4m33.7-5.2c29-2.5 54.4-9.5 79-21.8 41.4-20.6 75.5-54.8 96.1-96.1 16.9-34 24.9-74.2 21.8-109.9-3.7-42.4-17.6-79.2-42.3-112-34.4-45.8-84.7-75.5-141.4-83.5-25.2-3.6-49.5-2.5-75.2 3.3-49.6 11.2-93.7 40-124.4 81.2-21.3 28.7-35 62-40.1 97.8-2.8 19.4-2.8 38 0 57.4 8.1 56.7 37.7 107 83.5 141.4 11.1 8.3 21 14.5 33 20.5 34.1 16.9 74.4 24.8 110 21.7" style="fill:#b1b1b1"/><path d="M253.9 367.3c-2.9-.7-4.3-1.4-6-2.9-2.9-2.6-3.9-4.9-3.9-8.9 0-3.9 1-6.3 3.9-8.9 6-5.4 15.8-3.1 19 4.3 1.1 2.6 1 7-.3 9.6-1.2 2.4-3.7 4.9-6 5.8-1.7.8-5.3 1.3-6.7 1m68.4-29.3c-9.2-3.4-11-15.4-3.1-21.1 8.1-5.9 20 1 18.8 11-.5 4-3 7.6-6.7 9.5-2.4 1.2-6.6 1.5-9 .6m-139.8-.2c-3.4-1.1-6.2-3.5-7.9-6.7-1.3-2.4-1.2-7.7.2-10.2 1.2-2.3 3.9-4.8 6.3-5.8 2.6-1.1 7-1 9.6.3 4.2 2.1 6.7 6 6.7 10.7 0 6.5-4.7 11.5-11 11.9-1.5.1-3.3 0-3.9-.2m43.2-22.8c-2.4-.8-4.1-1.9-5.8-3.8-5.4-6-3.1-15.8 4.3-19 2.6-1.1 7-1 9.6.3 2.5 1.2 4.9 3.7 5.9 6.2.9 2.1 1 6.3.1 8.8-.8 2.3-3.5 5.3-5.9 6.6-1.9 1-6.3 1.5-8.2.9m54.6 0c-2.4-.8-4.1-1.9-5.8-3.8-5.4-6-3.1-15.8 4.3-19 2.6-1.1 7-1 9.6.3 2.5 1.2 4.9 3.7 5.9 6.2.9 2.1 1 6.3.1 8.8-.8 2.3-3.5 5.3-5.9 6.6-1.9 1-6.3 1.5-8.2.9m-27.6-47c-2.2-.7-4-1.9-5.5-3.5-2.5-2.7-3.2-4.5-3.2-8.5 0-3.9.7-5.8 3.2-8.5s4.8-3.6 8.8-3.6 6.3.9 8.8 3.6 3.2 4.5 3.2 8.5c0 2.5-.3 4-.9 5.2-1.2 2.3-3.5 4.7-5.7 5.8-1.9 1.1-6.7 1.6-8.7 1m99.7 0c-2.4-.8-4.1-1.9-5.8-3.8-5.4-6-3.1-15.8 4.3-19 2.6-1.1 7-1 9.6.3 2.5 1.2 4.9 3.7 5.9 6.2.9 2.1 1 6.3.1 8.8-.8 2.3-3.5 5.3-5.9 6.6-1.9 1-6.3 1.5-8.2.9m-198.9-.7c-8.9-2.5-11.8-13.8-5.1-20.1 2.6-2.5 4.5-3.2 8.2-3.2 3.9 0 6.3 1 8.9 3.9 2.1 2.3 3 4.7 2.9 8.2 0 7.6-7.5 13.2-14.9 11.2m42.9-.4c-2.3-1.2-5.2-4.1-6.1-6.4-1-2.4-.9-7.1.3-9.5 1.1-2.3 4.1-5.2 6.4-6.2 2.4-1 7.1-.9 9.5.3 2.3 1.1 5.2 4.1 6.2 6.4 1 2.4.9 7.1-.3 9.5-1.1 2.3-4.1 5.2-6.4 6.2-2.5 1.1-7.2.9-9.6-.3m109-.3c-11.1-5.6-7-22.5 5.5-22.4 3.7 0 6.1 1.1 8.6 3.9 2.2 2.4 3.2 4.9 3.2 8s-1 5.6-3.2 8c-2.5 2.8-5 3.9-8.6 3.9-2.3-.1-3.5-.4-5.5-1.4m-79.7-46.3c-8.9-2.5-11.8-13.8-5.1-20.1 2.6-2.5 4.5-3.2 8.2-3.2 3.9 0 6.3 1 8.9 3.9 2.1 2.3 3 4.7 2.9 8.2-.1 7.6-7.6 13.3-14.9 11.2m54.5 0c-8-2.2-11.3-12-6.4-18.5 5.7-7.5 16.1-6.4 20.5 2 1.3 2.4 1.2 7.7-.2 10.2-2.6 5.1-8.6 7.8-13.9 6.3m43.2-22.8c-8-2.2-11.3-12-6.4-18.5 5.7-7.5 16.1-6.4 20.5 2 1.3 2.4 1.2 7.7-.2 10.2-2.6 5.2-8.6 7.8-13.9 6.3m-140.6-.5c-8.7-2.4-11.6-13-5.4-19.7 2.5-2.6 4.8-3.6 8.6-3.6 3.9 0 6.3 1.1 8.8 3.8 2.2 2.5 3 4.7 3 8 .1 8-7.3 13.6-15 11.5m70.3-28.9c-8-2.2-11.3-12-6.4-18.5 5.7-7.5 16.1-6.4 20.5 2 1.3 2.4 1.2 7.7-.2 10.2-2.6 5.1-8.5 7.8-13.9 6.3" style="fill:#884717"/></svg>

main.py

@@ -2,30 +2,32 @@ from __future__ import annotations
 
 import asyncio
 import logging
-import os
 import sys
 
 import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
 
-from src.consts import LOG_LEVEL, VERSION
+from src.consts import HOST, LOG_LEVEL, PORT, VERSION
 from src.endpoints import health_check, router
 from src.middlewares import LogRequest
-from src.utils import get_camoufox, logger
+from src.owui import router as owui_router
+from src.utils import get_browser, logger
 
 logger.info("Using version %s", VERSION)
+logger.info("Log level set to %s", logging.getLevelName(LOG_LEVEL))
 
 app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL)
 app.add_middleware(GZipMiddleware)
 app.add_middleware(LogRequest)
 
 app.include_router(router=router)
+app.include_router(router=owui_router)
 
 
 async def init():
     """Initialize the application."""
-    async for browser in get_camoufox():
+    async for browser in get_browser():
         await health_check(browser)
 
 
@@ -38,5 +40,4 @@ if __name__ == "__main__":
         loop.run_until_complete(init())
         logger.info("Initialization complete.")
     else:
-        host = os.getenv("HOST", "0.0.0.0")  # noqa: S104
-        uvicorn.run(app, host=host, port=8191)
+        uvicorn.run(app, host=HOST, port=PORT)

pyproject.toml

@@ -2,34 +2,34 @@
 [project]
 authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
 license = { text = "LICENSE" }
-requires-python = "<4.0,>=3.12"
+requires-python = "==3.14.*"
 name = "Byparr"
 version = "0.1.0"
 description = "API for getting cookies for Cloudflare challenges"
 readme = "README.md"
 dependencies = [
-    "camoufox[geoip]==0.4.*",
-    "fastapi[standard]==0.116.*",
+    "fastapi[standard]==0.141.*",
+    "invisible-playwright>=0.6.1",
+    "playwright==1.60.*",
     "playwright-captcha==0.1.*",
-    "pydantic==2.11.*",
+    "pydantic==2.*",
+    "pydantic-settings==2.*",
+    "trafilatura==2.2.*",
 ]
 urls = { repository = "https://github.com/ThePhaseless/Byparr" }
-
 [dependency-groups]
+
 test = [
-    "httpx==0.28.*",
-    "pytest==8.4.*",
-    "pytest-asyncio==1.1.*",
-    "pytest-progress==1.3.*",
+    "httpx2==2.9.*",
+    "pytest==9.1.*",
+    "pytest-asyncio==1.4.*",
     "pytest-retry==1.7.*",
     "pytest-xdist==3.8.*",
     "setuptools>=80.9.0",
 ]
-dev = ["deptry==0.23.*", "ruff==0.12.*"]
-
+dev = ["deptry==0.25.*", "ruff==0.16.*"]
 [tool.deptry.per_rule_ignores]
 DEP002 = ["pyautogui"]
-
 [tool.ruff.lint]
 ignore = [
     "D203",

renovate.json

@@ -2,6 +2,16 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:recommended"],
   "forkProcessing": "enabled",
+  "lockFileMaintenance": {
+    "enabled": true,
+    "rebaseWhen": "behind-base-branch",
+    "branchTopic": "lock-file-maintenance",
+    "commitMessageAction": "Lock file maintenance",
+    "schedule": ["before 4am on monday"],
+    "prBodyDefinitions": {
+      "Change": "All locks refreshed"
+    }
+  },
   "packageRules": [
     {
       "automerge": true,

src/consts.py

@@ -1,23 +1,49 @@
 import logging
-import os
-from pathlib import Path
+import sys
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from playwright_captcha import CaptchaType
-from playwright_captcha.utils.camoufox_add_init_script.add_init_script import (
-    get_addon_path,
-)
-
-LOG_LEVEL = logging.getLevelNamesMapping()[os.getenv("LOG_LEVEL", "INFO").upper()]
-
-VERSION = os.getenv("VERSION", "unknown").removeprefix("v")
-
-ADDON_PATH = str(Path(get_addon_path()).absolute())
-MAX_ATTEMPTS = 2**10
 
 
-PROXY_SERVER = os.getenv("PROXY_SERVER")
-PROXY_USERNAME = os.getenv("PROXY_USERNAME")
-PROXY_PASSWORD = os.getenv("PROXY_PASSWORD")
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    log_level: str = "INFO"
+    version: str = "unknown"
+
+    max_attempts: int = sys.maxsize
+
+    proxy_server: str | None = None
+    proxy_username: str | None = None
+    proxy_password: str | None = None
+
+    host: str = "0.0.0.0"  # noqa: S104
+    port: int = 8191
+
+    block_media: bool = False
+    return_only_cookies: bool = False
+    owui_api_key: str | None = None
+
+
+settings = Settings()
+
+LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()]
+VERSION = settings.version.removeprefix("v")
+
+MAX_ATTEMPTS = settings.max_attempts
+
+PROXY_SERVER = settings.proxy_server
+PROXY_USERNAME = settings.proxy_username
+PROXY_PASSWORD = settings.proxy_password
+
+HOST = settings.host
+PORT = settings.port
+
+BLOCK_MEDIA = settings.block_media
+RETURN_ONLY_COOKIES = settings.return_only_cookies
+
+OWUI_API_KEY = settings.owui_api_key
 
 CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = {
     # Cloudflare

src/endpoints.py

@@ -1,3 +1,4 @@
+import base64
 import time
 import warnings
 from asyncio import wait_for
@@ -6,7 +7,7 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
-from httpx import TimeoutException
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright_captcha import CaptchaType
 
 from src.consts import CHALLENGE_TITLES
@@ -16,14 +17,18 @@ from src.models import (
     LinkResponse,
     Solution,
 )
-from src.utils import CamoufoxDepType, get_camoufox, logger
+from src.utils import BrowserDepClass, TimeoutTimer, get_browser, logger
 
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
 
 router = APIRouter()
 
-CamoufoxDep = Annotated[CamoufoxDepType, Depends(get_camoufox)]
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+CSP_HEADERS = frozenset(
+    {"content-security-policy", "content-security-policy-report-only"}
+)
 
 
 @router.get("/", include_in_schema=False)
@@ -34,7 +39,7 @@ def read_root():
 
 
 @router.get("/health")
-async def health_check(sb: CamoufoxDep):
+async def health_check(sb: BrowserDep):
     """Health check endpoint."""
     health_check_request = await read_item(
         LinkRequest.model_construct(url="https://google.com"),
@@ -51,48 +56,116 @@ async def health_check(sb: CamoufoxDep):
 
 
 @router.post("/v1")
-async def read_item(request: LinkRequest, dep: CamoufoxDep) -> LinkResponse:
+async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     """Handle POST requests."""
     start_time = int(time.time() * 1000)
 
-    request.url = request.url.replace('"', "").strip()
-    page_request = await dep.page.goto(request.url)
-    await dep.page.wait_for_load_state(state="domcontentloaded")
-    await dep.page.wait_for_load_state("networkidle")
+    timer = TimeoutTimer(duration=request.max_timeout)
 
-    if await dep.page.title() in CHALLENGE_TITLES:
-        logger.info("Challenge detected, attempting to solve...")
-        # Solve the captcha
-        remaining_timeout = (
-            request.max_timeout - (int(time.time()) * 1000 - start_time) / 1000
+    request.url = request.url.replace('"', "").strip()
+
+    if request.block_media:
+        async def block_media_route(route) -> None:
+            if route.request.resource_type in ("image", "media", "font"):
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await dep.page.route("**/*", block_media_route)
+
+    final_url: str | None = None
+
+    async def strip_csp_route(route) -> None:
+        nonlocal final_url
+        if route.request.resource_type != "document":
+            await route.continue_()
+            return
+        response = await route.fetch()
+        if route.request.frame == dep.page.main_frame:
+            final_url = response.url
+        await route.fulfill(
+            response=response,
+            headers={
+                key: value
+                for key, value in response.headers.items()
+                if key.lower() not in CSP_HEADERS
+            },
         )
-        logger.debug(
-            "Remaining timeout for solving the challenge: %d ms", remaining_timeout
+
+    await dep.page.route("**/*", strip_csp_route)
+
+    try:
+        page_request = await dep.page.goto(
+            request.url, timeout=timer.remaining() * 1000
         )
-        try:
+        status = page_request.status if page_request else HTTPStatus.OK
+        await dep.page.wait_for_load_state(
+            state="domcontentloaded", timeout=timer.remaining() * 1000
+        )
+
+        if await dep.page.title() in CHALLENGE_TITLES:
+            logger.info("Challenge detected, attempting to solve...")
+            # Solve the captcha
             await wait_for(
                 dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
                     captcha_container=dep.page,
                     captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
+                    wait_checkbox_attempts=1,
+                    wait_checkbox_delay=0.5,
                 ),
-                timeout=remaining_timeout,
+                timeout=timer.remaining(),
             )
-        except TimeoutException as e:
-            logger.error("Failed to solve challenge: %s", e)
-            return LinkResponse.invalid(url=request.url)
-        logger.debug("Challenge solved successfully.")
+            status = HTTPStatus.OK
+            logger.debug("Challenge solved successfully.")
+        else:
+            try:
+                await dep.page.wait_for_load_state(
+                    "networkidle", timeout=timer.remaining() * 1000
+                )
+            except PlaywrightTimeoutError:
+                logger.info(
+                    "networkidle timed out after domcontentloaded; continuing with loaded page"
+                )
+    except (TimeoutError, PlaywrightTimeoutError) as e:
+        logger.error("Timed out while loading the page or solving the challenge")
+        raise HTTPException(
+            status_code=408,
+            detail="Timed out while loading the page or solving the challenge",
+        ) from e
 
     cookies = await dep.context.cookies()
 
+    content_type = "text/html"
+    response_content = ""
+
+    if request.return_only_cookies:
+        response_content = ""
+    elif page_request and page_request.headers.get("content-type", "").startswith(
+        "application/pdf"
+    ):
+        content_type = "application/pdf"
+        try:
+            fetch_response = await dep.page.request.fetch(dep.page.url)
+            response_content = base64.b64encode(
+                await fetch_response.body()
+            ).decode("ascii")
+        except Exception:
+            logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
+            content_type = "text/html"
+            response_content = await dep.page.content()
+    else:
+        response_content = await dep.page.content()
+
     return LinkResponse(
         message="Success",
         solution=Solution(
             user_agent=await dep.page.evaluate("navigator.userAgent"),
-            url=dep.page.url,
-            status=page_request.status if page_request else HTTPStatus.OK,
+            url=final_url if final_url is not None else dep.page.url,
+            status=status,
             cookies=cookies,
             headers=page_request.headers if page_request else {},
-            response=await dep.page.content(),
+            response=response_content,
+            content_type=content_type,
         ),
         start_timestamp=start_time,
     )

src/middlewares.py

@@ -25,6 +25,6 @@ class LogRequest(BaseHTTPMiddleware):
         if response.status_code == HTTPStatus.OK:
             logger.info(f"Done {request_body.url} in {process_time:.2f}s")
         else:
-            logger.info(f"Failed {request_body.url} in {process_time:.2f}s")
+            logger.warning(f"Failed {request_body.url} in {process_time:.2f}s")
 
         return response

src/models.py

@@ -5,13 +5,17 @@ from http.client import INTERNAL_SERVER_ERROR
 from typing import Any
 
 from playwright.sync_api import Cookie
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic.alias_generators import to_camel
 
 from src import consts
 
+MS_PER_SECOND = 1000
+
 
 class LinkRequest(BaseModel):
+    model_config = {"populate_by_name": True}
+
     cmd: str = Field(
         default="request.get",
         description="Type of request, currently only supports GET requests. This string is purely for compatibility with FlareSolverr.",
@@ -19,13 +23,36 @@ class LinkRequest(BaseModel):
     url: str = Field(pattern=r"^https?://", default="https://")
     max_timeout: int = Field(
         default=60,
-        description="Maximum timeout in seconds for resolving the anti-bot challenge.",
+        alias="maxTimeout",
+        description=(
+            "Maximum timeout for resolving the anti-bot challenge. Values below 1000 "
+            "are treated as seconds; values of 1000 or more as milliseconds, matching "
+            "FlareSolverr's maxTimeout parameter."
+        ),
     )
+    block_media: bool = Field(
+        default=consts.BLOCK_MEDIA,
+        alias="blockMedia",
+        description="Block image, media, and font resources from loading.",
+    )
+    return_only_cookies: bool = Field(
+        default=consts.RETURN_ONLY_COOKIES,
+        alias="returnOnlyCookies",
+        description="Return only cookies, skip the page HTML content in the response.",
+    )
+
+    @field_validator("max_timeout")
+    @classmethod
+    def normalize_max_timeout(cls, value: int) -> int:
+        """Normalize FlareSolverr-style millisecond values to seconds."""
+        if value >= MS_PER_SECOND:
+            return value // MS_PER_SECOND
+        return value
 
 
 class HealthcheckResponse(BaseModel):
     model_config = {"alias_generator": to_camel, "populate_by_name": True}
-    msg: str = "Byparr is ready!"
+    msg: str = "Byparr is working!"
     version: str = consts.VERSION
     user_agent: str
 
@@ -38,6 +65,7 @@ class Solution(BaseModel):
     user_agent: str = ""
     headers: dict[str, Any] = {}
     response: str = ""
+    content_type: str = Field(default="text/html", alias="contentType")
 
 
 class LinkResponse(BaseModel):

src/owui.py

@@ -0,0 +1,76 @@
+"""Open WebUI external web loader endpoint: POST /load."""
+
+from __future__ import annotations
+
+from hmac import compare_digest
+from typing import Annotated
+
+import trafilatura
+from fastapi import APIRouter, Depends, Header, HTTPException
+from playwright.async_api import Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from pydantic import BaseModel
+
+from src.consts import OWUI_API_KEY
+from src.utils import BrowserDepClass, get_browser, logger
+
+router = APIRouter(tags=["Open WebUI"])
+
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+
+class LoadRequest(BaseModel):
+    urls: list[str]
+
+
+class LoadResult(BaseModel):
+    page_content: str
+    metadata: dict[str, str]
+
+
+def require_auth(authorization: Annotated[str | None, Header()] = None) -> None:
+    """Enforce a bearer token on /load when OWUI_API_KEY is set."""
+    if not OWUI_API_KEY:
+        return
+    if authorization is None or not compare_digest(
+        authorization.encode(), f"Bearer {OWUI_API_KEY}".encode()
+    ):
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+async def _extract_content(page: Page) -> str:
+    """Return the page's main article text, falling back to visible text."""
+    article = trafilatura.extract(await page.content())
+    if article:
+        return article
+    result = await page.evaluate("() => document.body ? document.body.innerText : ''")
+    return "\n".join(line.strip() for line in result.splitlines() if line.strip())
+
+
+@router.post("/load", response_model=list[LoadResult])
+async def load_urls(
+    request: LoadRequest,
+    _auth: Annotated[None, Depends(require_auth)],
+    dep: BrowserDep,
+) -> list[LoadResult]:
+    """
+    Fetch URLs through the anti-bot browser and return their text content.
+
+    Each URL is fetched sequentially; a failing URL yields empty
+    page_content so Open WebUI's RAG pipeline degrades gracefully.
+    """
+    results: list[LoadResult] = []
+    for url in request.urls:
+        try:
+            await dep.page.goto(url, timeout=60_000)
+            await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            try:
+                await dep.page.wait_for_load_state("networkidle", timeout=15_000)
+            except PlaywrightTimeoutError:
+                logger.debug("networkidle timed out for %s; extracting anyway", url)
+            content = await _extract_content(dep.page)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to load %s: %s", url, exc)
+            content = ""
+        results.append(LoadResult(page_content=content, metadata={"source": url}))
+    return results

src/utils.py

@@ -1,16 +1,18 @@
 import logging
+import time
 from collections.abc import AsyncGenerator
-from typing import NamedTuple, cast
+from typing import Annotated, NamedTuple, cast
 
-from camoufox import AsyncCamoufox
+from fastapi import Header
+from invisible_playwright.async_api import InvisiblePlaywright
 from playwright.async_api import Browser, BrowserContext, Page
 from playwright_captcha import (
     ClickSolver,
     FrameworkType,
 )
+from pydantic import BaseModel, Field
 
 from src.consts import (
-    ADDON_PATH,
     LOG_LEVEL,
     MAX_ATTEMPTS,
     PROXY_PASSWORD,
@@ -18,9 +20,13 @@ from src.consts import (
     PROXY_USERNAME,
 )
 
-solver_logger = logging.getLogger("playwright_captcha.solvers")
+solver_logger = logging.getLogger("playwright_captcha")
 solver_logger.handlers.clear()
-solver_logger.handlers.append(logging.NullHandler())
+if LOG_LEVEL == logging.DEBUG:
+    solver_logger.addHandler(logging.StreamHandler())
+    solver_logger.setLevel(LOG_LEVEL)
+else:
+    solver_logger.handlers.append(logging.NullHandler())
 
 logger = logging.getLogger("uvicorn.error")
 logger.setLevel(LOG_LEVEL)
@@ -28,44 +34,76 @@ if len(logger.handlers) == 0:
     logger.addHandler(logging.StreamHandler())
 
 
-class CamoufoxDepType(NamedTuple):
+class TimeoutTimer(BaseModel):
+    duration: int  # in seconds
+    start_time: float = Field(default_factory=time.perf_counter)
+
+    def remaining(self) -> float:
+        """Get remaining time in seconds."""
+        return max(0, self.duration - (time.perf_counter() - self.start_time))
+
+
+class BrowserDepClass(NamedTuple):
     page: Page
     solver: ClickSolver
     context: BrowserContext
 
 
-async def get_camoufox() -> AsyncGenerator[CamoufoxDepType, None]:
-    """Get Camoufox instance."""
-    proxy_config = (
-        {
+async def get_browser(
+    x_proxy_server: Annotated[
+        str | None,
+        Header(
+            alias="X-Proxy-Server",
+            description="Override proxy server for this request in protocol://host:port format.",
+        ),
+    ] = None,
+    x_proxy_username: Annotated[
+        str | None,
+        Header(
+            alias="X-Proxy-Username",
+        ),
+    ] = None,
+    x_proxy_password: Annotated[
+        str | None,
+        Header(
+            alias="X-Proxy-Password",
+        ),
+    ] = None,
+) -> AsyncGenerator[BrowserDepClass]:
+    """Get InvisiblePlaywright browser instance."""
+    header_server = x_proxy_server
+    header_username = x_proxy_username
+    header_password = x_proxy_password
+
+    proxy_config = None
+
+    if header_server:
+        proxy_config = {
+            "server": header_server,
+            "username": header_username,
+            "password": header_password,
+        }
+    elif PROXY_SERVER:
+        proxy_config = {
             "server": PROXY_SERVER,
             "username": PROXY_USERNAME,
             "password": PROXY_PASSWORD,
         }
-        if PROXY_SERVER
-        else None
-    )
 
-    async with AsyncCamoufox(
-        main_world_eval=True,
-        addons=[ADDON_PATH],
-        geoip=True,
-        proxy=proxy_config,
-        locale="en-US",
+    async with InvisiblePlaywright(
         headless=True,
+        proxy=proxy_config,
         humanize=True,
-        i_know_what_im_doing=True,
-        config={"forceScopeAccess": True},  # add this when creating Camoufox instance
-        disable_coop=True,  # add this when creating Camoufox instance
+        locale="auto",
     ) as browser_raw:
-        # Cast to Browser since AsyncCamoufox always returns a Browser, not BrowserContext
+        # InvisiblePlaywright yields a Browser instance
         browser = cast("Browser", browser_raw)
         context = await browser.new_context()
         page = await context.new_page()
         async with ClickSolver(
-            framework=FrameworkType.CAMOUFOX,
+            framework=FrameworkType.PLAYWRIGHT,
             page=page,
             max_attempts=MAX_ATTEMPTS,
             attempt_delay=1,
         ) as solver:
-            yield CamoufoxDepType(page, solver, context)
+            yield BrowserDepClass(page, solver, context)

tests/main_test.py

@@ -1,12 +1,17 @@
 from http import HTTPStatus
 from json import JSONDecodeError
+from unittest.mock import AsyncMock, MagicMock
 
-import httpx
+import httpx2
 import pytest
+from fastapi import HTTPException
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
+from src.endpoints import read_item
 from src.models import LinkRequest
+from src.utils import BrowserDepClass
 
 client = TestClient(app)
 
@@ -27,7 +32,7 @@ def test_bypass(website: str):
 
     This test is skipped if the website is not reachable or does not have cloudflare/DDOS-GUARD.
     """
-    test_request = httpx.get(
+    test_request = httpx2.get(
         website,
     )
     if (
@@ -47,6 +52,9 @@ def test_bypass(website: str):
         json=LinkRequest.model_construct(url=website, cmd="request.get").model_dump(),
     )
 
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip(f"Skipping {website} - timed out (upstream issue)")
+
     assert response.status_code == HTTPStatus.OK
 
 
@@ -59,3 +67,89 @@ def test_health_check():
     """
     response = client.get("/health")
     assert response.status_code == HTTPStatus.OK
+
+
+def test_pdf_handling():
+    """Tests that PDF URLs return the raw PDF bytes, not the Firefox viewer HTML."""
+    pdf_url = "https://mondaymandala.com/wp-content/uploads/Mickey-And-Minnie-Mouse-Holding-An-Easter-Egg-Basket-Coloring-Page-For-Kids.pdf"
+    response = client.post(
+        "/v1",
+        json=LinkRequest.model_construct(url=pdf_url, cmd="request.get").model_dump(),
+    )
+    if response.status_code == HTTPStatus.REQUEST_TIMEOUT:
+        pytest.skip("Skipping PDF test - timed out (upstream issue)")
+    assert response.status_code == HTTPStatus.OK
+    solution = response.json()["solution"]
+    if solution.get("contentType") != "application/pdf":
+        pytest.skip("Skipping PDF test - PDF bytes could not be fetched (upstream issue)")
+    assert solution["response"]  # non-empty base64
+    import base64
+
+    decoded = base64.b64decode(solution["response"])
+    assert decoded[:5] == b"%PDF-"
+
+
+@pytest.mark.parametrize(
+    ("payload", "expected"),
+    [
+        ({"max_timeout": 60}, 60),  # native API: seconds
+        ({"maxTimeout": 60}, 60),  # FlareSolverr alias, seconds-range value
+        ({"maxTimeout": 60000}, 60),  # FlareSolverr alias: milliseconds
+        ({"maxTimeout": 55000}, 55),
+        ({"maxTimeout": 1000}, 1),
+        ({}, 60),  # default
+    ],
+)
+def test_max_timeout_normalization(payload: dict, expected: int):
+    """MaxTimeout must accept FlareSolverr's milliseconds while keeping seconds."""
+    request = LinkRequest(url="https://example.com", **payload)
+    assert request.max_timeout == expected
+
+
+def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
+    """Build a browser dependency triple backed by mocks."""
+    page = AsyncMock()
+    page.url = "https://example.test/login"
+    page.goto.return_value = MagicMock(
+        status=HTTPStatus.OK, headers={"content-type": "text/html"}
+    )
+    page.title.return_value = "Login"
+    page.evaluate.return_value = "UnitTestBrowser/1.0"
+    page.content.return_value = "<html><title>Login</title></html>"
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        """Fail the wait when asked for a configured state."""
+        if state in (fail_states or set()):
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+
+    context = AsyncMock()
+    context.cookies.return_value = []
+    return BrowserDepClass(page=page, solver=AsyncMock(), context=context)
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_after_domcontentloaded_returns_content():
+    """Pages that never go idle after DOM load must still return their content."""
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        fake_dep(fail_states={"networkidle"}),
+    )
+
+    assert response.status == "ok"
+    assert response.solution.status == HTTPStatus.OK
+    assert response.solution.response == "<html><title>Login</title></html>"
+
+
+@pytest.mark.asyncio
+async def test_domcontentloaded_timeout_returns_408():
+    """Fatal timeouts during initial page load still return a controlled 408."""
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login"),
+            fake_dep(fail_states={"domcontentloaded"}),
+        )
+
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT

tests/owui_test.py

@@ -0,0 +1,112 @@
+from http import HTTPStatus
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from starlette.testclient import TestClient
+
+from main import app
+from src.owui import LoadRequest, load_urls
+from src.utils import BrowserDepClass
+
+client = TestClient(app)
+
+
+def test_owui_load_basic():
+    """/load returns one result per URL with the expected shape."""
+    response = client.post("/load", json={"urls": ["https://example.com"]})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"]
+    assert results[0]["metadata"] == {"source": "https://example.com"}
+
+
+def test_owui_load_multiple_urls():
+    """/load returns one result per URL, in order."""
+    urls = ["https://example.com", "https://example.org"]
+    response = client.post("/load", json={"urls": urls})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert [r["metadata"]["source"] for r in results] == urls
+
+
+def test_owui_load_invalid_url_graceful():
+    """Unreachable URLs yield empty page_content instead of an error."""
+    response = client.post(
+        "/load", json={"urls": ["https://this-domain-does-not-exist-12345.invalid"]}
+    )
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"] == ""
+
+
+@pytest.mark.parametrize(
+    "headers",
+    [None, {"Authorization": "Bearer wrong-key"}],
+)
+def test_owui_load_rejects_missing_or_wrong_key(headers):
+    """/load returns 401 without a valid bearer token when a key is set."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load", json={"urls": ["https://example.com"]}, headers=headers
+        )
+        assert response.status_code == HTTPStatus.UNAUTHORIZED
+
+
+def test_owui_load_accepts_valid_key():
+    """/load succeeds with the configured bearer token."""
+    with patch("src.owui.OWUI_API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load",
+            json={"urls": ["https://example.com"]},
+            headers={"Authorization": "Bearer test-secret-key"},
+        )
+        assert response.status_code == HTTPStatus.OK
+
+
+ARTICLE_HTML = """<html><head><title>Test</title></head><body>
+<article><h1>Example Title</h1><p>This is the main article body with enough words for
+trafilatura to consider it real content rather than boilerplate.</p></article>
+<nav><a href="/x">nav link</a></nav>
+</body></html>"""
+
+
+def fake_dep(*, html: str = ARTICLE_HTML) -> BrowserDepClass:
+    """Browser dependency whose page loads HTML but never reaches networkidle."""
+    page = AsyncMock()
+    page.goto.return_value = MagicMock()
+    page.content.return_value = html
+    page.evaluate.return_value = "line one\n\nline two"
+
+    def wait_for_load_state(state: str, **_kwargs: object) -> None:
+        if state == "networkidle":
+            message = "load state wait timed out"
+            raise PlaywrightTimeoutError(message)
+
+    page.wait_for_load_state.side_effect = wait_for_load_state
+    return BrowserDepClass(page=page, solver=AsyncMock(), context=AsyncMock())
+
+
+@pytest.mark.asyncio
+async def test_networkidle_timeout_still_extracts_content():
+    """A page that never reaches networkidle still yields its article text."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]), None, fake_dep()
+    )
+    assert results[0].page_content == (
+        "Example TitleThis is the main article body with enough words for "
+        "trafilatura to consider it real content rather than boilerplate."
+    )
+
+
+@pytest.mark.asyncio
+async def test_extraction_falls_back_to_innertext():
+    """Pages trafilatura cannot score fall back to the rendered innerText."""
+    results = await load_urls(
+        LoadRequest(urls=["https://example.test"]),
+        None,
+        fake_dep(html="<html><body></body></html>"),
+    )
+    assert results[0].page_content == "line one\nline two"