diff --git a/Dockerfile b/Dockerfile index 9b02061..8fc3fe0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,10 @@ ENV GITHUB_BUILD=${GITHUB_BUILD}\ UV_LINK_MODE=copy \ PORT=8191 \ XDG_CACHE_HOME=/cache \ - HOME=/tmp + HOME=/tmp \ + # Optional: set to require bearer token on /load requests. + # Must match EXTERNAL_WEB_LOADER_API_KEY in Open WebUI. + OWUI_API_KEY="" RUN apt-get update &&\ apt-get install -y --no-install-recommends curl ca-certificates git tini &&\ diff --git a/README.md b/README.md index 5e52141..74898c4 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ | `PROXY_SERVER` | None | Proxy to use in format: `protocol://host:port`. | | `PROXY_USERNAME` | None | Username for proxy authentication. | | `PROXY_PASSWORD` | None | Password for proxy authentication. | +| `OWUI_API_KEY` | None | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI. | ## Proxy Recommendation @@ -65,6 +66,20 @@ Once running, open: - `http://localhost:8191/docs` - `http://localhost:8191/` (redirects to `/docs`) +### Open WebUI Integration + +Byparr can serve as an external web loader for [Open WebUI](https://github.com/open-webui/open-webui), allowing it to fetch web content through Byparr's anti-bot bypassing capabilities. + +Configure Open WebUI with these environment variables: + +```bash +WEB_LOADER_ENGINE=external +EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load +EXTERNAL_WEB_LOADER_API_KEY=your-secret-key # Optional, must match OWUI_API_KEY +``` + +The `/load` endpoint accepts `POST` requests with `{"urls": ["https://..."]}` and returns extracted text content for RAG pipelines. + ## Troubleshooting ### Docker troubleshooting diff --git a/main.py b/main.py index d1b9b1c..5e849fb 100644 --- a/main.py +++ b/main.py @@ -11,6 +11,7 @@ from fastapi.middleware.gzip import GZipMiddleware from src.consts import HOST, LOG_LEVEL, PORT, VERSION from src.endpoints import health_check, router from src.middlewares import LogRequest +from src.owui import router as owui_router from src.utils import get_browser, logger logger.info("Using version %s", VERSION) @@ -21,6 +22,7 @@ app.add_middleware(GZipMiddleware) app.add_middleware(LogRequest) app.include_router(router=router) +app.include_router(router=owui_router) async def init(): diff --git a/src/owui.py b/src/owui.py new file mode 100644 index 0000000..55e660b --- /dev/null +++ b/src/owui.py @@ -0,0 +1,114 @@ +""" +src/owui.py — Open WebUI external web loader endpoint. + +Implements the contract expected by Open WebUI's WEB_LOADER_ENGINE=external: + POST /load + Request: {"urls": ["https://..."]} + Response: [{"page_content": str, "metadata": {"source": str}}] + +Reuses Byparr's existing browser dependency injection so each request +gets a properly-initialised browser context with challenge solving. +Uses document.body.innerText for content extraction. + +Configure in Open WebUI: + WEB_LOADER_ENGINE=external + EXTERNAL_WEB_LOADER_URL=http://byparr:8191/load + EXTERNAL_WEB_LOADER_API_KEY= +""" + +from __future__ import annotations + +import os +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, Header, HTTPException +from pydantic import BaseModel + +from src.utils import BrowserDepClass, get_browser, logger + +_API_KEY: str | None = os.getenv("OWUI_API_KEY") or None + +router = APIRouter(tags=["Open WebUI"]) + +BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)] + + +# --------------------------------------------------------------------------- +# Models +# --------------------------------------------------------------------------- + + +class LoadRequest(BaseModel): + urls: list[str] + + +class LoadResult(BaseModel): + page_content: str + metadata: dict[str, Any] + + +# --------------------------------------------------------------------------- +# Content extraction +# --------------------------------------------------------------------------- + + +async def _extract_content(page: Any) -> str: + """ + Extract text content from the page using document.body.innerText. + Whitespace is collapsed for clean RAG chunking. + """ + result: str = await page.evaluate(""" + () => { + return document.body ? document.body.innerText : ""; + } + """) + lines = [line.strip() for line in result.splitlines() if line.strip()] + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Auth +# --------------------------------------------------------------------------- + + +def _check_auth(authorization: str | None) -> None: + if not _API_KEY: + return + if authorization != f"Bearer {_API_KEY}": + raise HTTPException(status_code=401, detail="Unauthorized") + + +# --------------------------------------------------------------------------- +# Endpoint +# --------------------------------------------------------------------------- + + +@router.post("/load", response_model=list[LoadResult]) +async def load_urls( + request: LoadRequest, + authorization: Annotated[str | None, Header()] = None, + dep: BrowserDep = None, # type: ignore[assignment] +) -> list[LoadResult]: + """ + Fetch URLs through InvisiblePlaywright, bypassing Cloudflare challenges. + Returns plain-text content for Open WebUI's RAG pipeline. + Failed URLs return empty page_content so the search degrades gracefully. + """ + _check_auth(authorization) + + results: list[LoadResult] = [] + + for url in request.urls: + try: + logger.info("OWUI loader fetching: %s", url) + await dep.page.goto(url, timeout=60_000) + await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000) + await dep.page.wait_for_load_state("networkidle", timeout=15_000) + content = await _extract_content(dep.page) + logger.debug("OWUI loader: %s -> %d chars", url, len(content)) + results.append(LoadResult(page_content=content, metadata={"source": url})) + except Exception as exc: # noqa: BLE001 + logger.warning("OWUI loader failed for %s: %s", url, exc) + results.append(LoadResult(page_content="", metadata={"source": url})) + + return results diff --git a/tests/owui_test.py b/tests/owui_test.py new file mode 100644 index 0000000..a51b9b7 --- /dev/null +++ b/tests/owui_test.py @@ -0,0 +1,81 @@ +from http import HTTPStatus +from unittest.mock import patch + +import pytest +from starlette.testclient import TestClient + +from main import app + +client = TestClient(app) + + +def test_owui_load_basic(): + """Test /load endpoint returns correct structure.""" + response = client.post("/load", json={"urls": ["https://example.com"]}) + assert response.status_code == HTTPStatus.OK + results = response.json() + assert len(results) == 1 + assert "page_content" in results[0] + assert "metadata" in results[0] + assert results[0]["metadata"]["source"] == "https://example.com" + + +def test_owui_load_multiple_urls(): + """Test /load endpoint handles multiple URLs.""" + urls = ["https://example.com", "https://example.org"] + response = client.post("/load", json={"urls": urls}) + assert response.status_code == HTTPStatus.OK + results = response.json() + assert len(results) == 2 + assert results[0]["metadata"]["source"] == urls[0] + assert results[1]["metadata"]["source"] == urls[1] + + +def test_owui_load_extracts_content(): + """Test /load endpoint extracts non-empty content from a real page.""" + response = client.post("/load", json={"urls": ["https://example.com"]}) + assert response.status_code == HTTPStatus.OK + results = response.json() + # example.com has minimal content but should have something + assert len(results[0]["page_content"]) > 0 + + +def test_owui_load_auth_required(): + """Test /load returns 401 when API key is set but not provided.""" + with patch("src.owui._API_KEY", "test-secret-key"): + response = client.post("/load", json={"urls": ["https://example.com"]}) + assert response.status_code == HTTPStatus.UNAUTHORIZED + + +def test_owui_load_auth_invalid(): + """Test /load returns 401 when API key is wrong.""" + with patch("src.owui._API_KEY", "test-secret-key"): + response = client.post( + "/load", + json={"urls": ["https://example.com"]}, + headers={"Authorization": "Bearer wrong-key"}, + ) + assert response.status_code == HTTPStatus.UNAUTHORIZED + + +def test_owui_load_auth_valid(): + """Test /load succeeds with correct API key.""" + with patch("src.owui._API_KEY", "test-secret-key"): + response = client.post( + "/load", + json={"urls": ["https://example.com"]}, + headers={"Authorization": "Bearer test-secret-key"}, + ) + assert response.status_code == HTTPStatus.OK + + +def test_owui_load_invalid_url_graceful(): + """Test /load returns empty content for unreachable URLs.""" + response = client.post( + "/load", json={"urls": ["https://this-domain-does-not-exist-12345.invalid"]} + ) + assert response.status_code == HTTPStatus.OK + results = response.json() + assert len(results) == 1 + assert results[0]["page_content"] == "" + assert "this-domain-does-not-exist" in results[0]["metadata"]["source"]