feat: add Open WebUI external web loader endpoint

Add /load endpoint for Open WebUI's WEB_LOADER_ENGINE=external integration. Uses document.body.innerText for content extraction. Configure in Open WebUI: WEB_LOADER_ENGINE=external EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load EXTERNAL_WEB_LOADER_API_KEY=<OWUI_API_KEY env var> Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-09-24 06:10:14 +01:00 · 2026-08-08 00:56:46 +02:00
parent f970d5cf0f
commit f76443cbda
5 changed files with 216 additions and 1 deletions
@@ -15,7 +15,10 @@ ENV GITHUB_BUILD=${GITHUB_BUILD}\
    UV_LINK_MODE=copy \
    PORT=8191 \
    XDG_CACHE_HOME=/cache \
-    HOME=/tmp
+    HOME=/tmp \
+    # Optional: set to require bearer token on /load requests.
+    # Must match EXTERNAL_WEB_LOADER_API_KEY in Open WebUI.
+    OWUI_API_KEY=""

 RUN apt-get update &&\
    apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
@@ -16,6 +16,7 @@
 | `PROXY_SERVER`       | None      | Proxy to use in format: `protocol://host:port`.                                                                                                                   |
 | `PROXY_USERNAME`     | None      | Username for proxy authentication.                                                                                                                                |
 | `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
+| `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |

 ## Proxy Recommendation

@@ -65,6 +66,20 @@ Once running, open:
 - `http://localhost:8191/docs`
 - `http://localhost:8191/` (redirects to `/docs`)

+### Open WebUI Integration
+
+Byparr can serve as an external web loader for [Open WebUI](https://github.com/open-webui/open-webui), allowing it to fetch web content through Byparr's anti-bot bypassing capabilities.
+
+Configure Open WebUI with these environment variables:
+
+```bash
+WEB_LOADER_ENGINE=external
+EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load
+EXTERNAL_WEB_LOADER_API_KEY=your-secret-key  # Optional, must match OWUI_API_KEY
+```
+
+The `/load` endpoint accepts `POST` requests with `{"urls": ["https://..."]}` and returns extracted text content for RAG pipelines.
+
 ## Troubleshooting

 ### Docker troubleshooting
@@ -11,6 +11,7 @@ from fastapi.middleware.gzip import GZipMiddleware
 from src.consts import HOST, LOG_LEVEL, PORT, VERSION
 from src.endpoints import health_check, router
 from src.middlewares import LogRequest
+from src.owui import router as owui_router
 from src.utils import get_browser, logger

 logger.info("Using version %s", VERSION)
@@ -21,6 +22,7 @@ app.add_middleware(GZipMiddleware)
 app.add_middleware(LogRequest)

 app.include_router(router=router)
+app.include_router(router=owui_router)


 async def init():
@@ -0,0 +1,114 @@
+"""
+src/owui.py — Open WebUI external web loader endpoint.
+
+Implements the contract expected by Open WebUI's WEB_LOADER_ENGINE=external:
+  POST /load
+    Request:  {"urls": ["https://..."]}
+    Response: [{"page_content": str, "metadata": {"source": str}}]
+
+Reuses Byparr's existing browser dependency injection so each request
+gets a properly-initialised browser context with challenge solving.
+Uses document.body.innerText for content extraction.
+
+Configure in Open WebUI:
+  WEB_LOADER_ENGINE=external
+  EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load
+  EXTERNAL_WEB_LOADER_API_KEY=<value of OWUI_API_KEY env var, if set>
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Header, HTTPException
+from pydantic import BaseModel
+
+from src.utils import BrowserDepClass, get_browser, logger
+
+_API_KEY: str | None = os.getenv("OWUI_API_KEY") or None
+
+router = APIRouter(tags=["Open WebUI"])
+
+BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
+
+
+# ---------------------------------------------------------------------------
+# Models
+# ---------------------------------------------------------------------------
+
+
+class LoadRequest(BaseModel):
+    urls: list[str]
+
+
+class LoadResult(BaseModel):
+    page_content: str
+    metadata: dict[str, Any]
+
+
+# ---------------------------------------------------------------------------
+# Content extraction
+# ---------------------------------------------------------------------------
+
+
+async def _extract_content(page: Any) -> str:
+    """
+    Extract text content from the page using document.body.innerText.
+    Whitespace is collapsed for clean RAG chunking.
+    """
+    result: str = await page.evaluate("""
+        () => {
+            return document.body ? document.body.innerText : "";
+        }
+    """)
+    lines = [line.strip() for line in result.splitlines() if line.strip()]
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Auth
+# ---------------------------------------------------------------------------
+
+
+def _check_auth(authorization: str | None) -> None:
+    if not _API_KEY:
+        return
+    if authorization != f"Bearer {_API_KEY}":
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+# ---------------------------------------------------------------------------
+# Endpoint
+# ---------------------------------------------------------------------------
+
+
+@router.post("/load", response_model=list[LoadResult])
+async def load_urls(
+    request: LoadRequest,
+    authorization: Annotated[str | None, Header()] = None,
+    dep: BrowserDep = None,  # type: ignore[assignment]
+) -> list[LoadResult]:
+    """
+    Fetch URLs through InvisiblePlaywright, bypassing Cloudflare challenges.
+    Returns plain-text content for Open WebUI's RAG pipeline.
+    Failed URLs return empty page_content so the search degrades gracefully.
+    """
+    _check_auth(authorization)
+
+    results: list[LoadResult] = []
+
+    for url in request.urls:
+        try:
+            logger.info("OWUI loader fetching: %s", url)
+            await dep.page.goto(url, timeout=60_000)
+            await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            await dep.page.wait_for_load_state("networkidle", timeout=15_000)
+            content = await _extract_content(dep.page)
+            logger.debug("OWUI loader: %s -> %d chars", url, len(content))
+            results.append(LoadResult(page_content=content, metadata={"source": url}))
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("OWUI loader failed for %s: %s", url, exc)
+            results.append(LoadResult(page_content="", metadata={"source": url}))
+
+    return results
@@ -0,0 +1,81 @@
+from http import HTTPStatus
+from unittest.mock import patch
+
+import pytest
+from starlette.testclient import TestClient
+
+from main import app
+
+client = TestClient(app)
+
+
+def test_owui_load_basic():
+    """Test /load endpoint returns correct structure."""
+    response = client.post("/load", json={"urls": ["https://example.com"]})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert "page_content" in results[0]
+    assert "metadata" in results[0]
+    assert results[0]["metadata"]["source"] == "https://example.com"
+
+
+def test_owui_load_multiple_urls():
+    """Test /load endpoint handles multiple URLs."""
+    urls = ["https://example.com", "https://example.org"]
+    response = client.post("/load", json={"urls": urls})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 2
+    assert results[0]["metadata"]["source"] == urls[0]
+    assert results[1]["metadata"]["source"] == urls[1]
+
+
+def test_owui_load_extracts_content():
+    """Test /load endpoint extracts non-empty content from a real page."""
+    response = client.post("/load", json={"urls": ["https://example.com"]})
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    # example.com has minimal content but should have something
+    assert len(results[0]["page_content"]) > 0
+
+
+def test_owui_load_auth_required():
+    """Test /load returns 401 when API key is set but not provided."""
+    with patch("src.owui._API_KEY", "test-secret-key"):
+        response = client.post("/load", json={"urls": ["https://example.com"]})
+        assert response.status_code == HTTPStatus.UNAUTHORIZED
+
+
+def test_owui_load_auth_invalid():
+    """Test /load returns 401 when API key is wrong."""
+    with patch("src.owui._API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load",
+            json={"urls": ["https://example.com"]},
+            headers={"Authorization": "Bearer wrong-key"},
+        )
+        assert response.status_code == HTTPStatus.UNAUTHORIZED
+
+
+def test_owui_load_auth_valid():
+    """Test /load succeeds with correct API key."""
+    with patch("src.owui._API_KEY", "test-secret-key"):
+        response = client.post(
+            "/load",
+            json={"urls": ["https://example.com"]},
+            headers={"Authorization": "Bearer test-secret-key"},
+        )
+        assert response.status_code == HTTPStatus.OK
+
+
+def test_owui_load_invalid_url_graceful():
+    """Test /load returns empty content for unreachable URLs."""
+    response = client.post(
+        "/load", json={"urls": ["https://this-domain-does-not-exist-12345.invalid"]}
+    )
+    assert response.status_code == HTTPStatus.OK
+    results = response.json()
+    assert len(results) == 1
+    assert results[0]["page_content"] == ""
+    assert "this-domain-does-not-exist" in results[0]["metadata"]["source"]