mirror of
https://github.com/ThePhaseless/Byparr.git
synced 2026-11-19 21:46:33 +00:00
- Fix I001: sort imports in src/consts.py - Fix PLC0415: move `import base64` to top of tests/main_test.py - Fix UP037: remove quotes from LinkResponse return annotation - Fix D213: correct multi-line docstring summary placement - Remove unused `# noqa: BLE001` in src/owui.py - Refactor read_item into helpers: setup_routes, load_page_and_solve, build_response_content, _fetch_pdf_content — resolves C901 and PLR0915 - Add CPY001, BLE001 to ruff ignore list
77 lines
2.6 KiB
Python
77 lines
2.6 KiB
Python
"""Open WebUI external web loader endpoint: POST /load."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from hmac import compare_digest
|
|
from typing import Annotated
|
|
|
|
import trafilatura
|
|
from fastapi import APIRouter, Depends, Header, HTTPException
|
|
from playwright.async_api import Page
|
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
|
from pydantic import BaseModel
|
|
|
|
from src.consts import OWUI_API_KEY
|
|
from src.utils import BrowserDepClass, get_browser, logger
|
|
|
|
router = APIRouter(tags=["Open WebUI"])
|
|
|
|
BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
|
|
|
|
|
|
class LoadRequest(BaseModel):
|
|
urls: list[str]
|
|
|
|
|
|
class LoadResult(BaseModel):
|
|
page_content: str
|
|
metadata: dict[str, str]
|
|
|
|
|
|
def require_auth(authorization: Annotated[str | None, Header()] = None) -> None:
|
|
"""Enforce a bearer token on /load when OWUI_API_KEY is set."""
|
|
if not OWUI_API_KEY:
|
|
return
|
|
if authorization is None or not compare_digest(
|
|
authorization.encode(), f"Bearer {OWUI_API_KEY}".encode()
|
|
):
|
|
raise HTTPException(status_code=401, detail="Unauthorized")
|
|
|
|
|
|
async def _extract_content(page: Page) -> str:
|
|
"""Return the page's main article text, falling back to visible text."""
|
|
article = trafilatura.extract(await page.content())
|
|
if article:
|
|
return article
|
|
result = await page.evaluate("() => document.body ? document.body.innerText : ''")
|
|
return "\n".join(line.strip() for line in result.splitlines() if line.strip())
|
|
|
|
|
|
@router.post("/load", response_model=list[LoadResult])
|
|
async def load_urls(
|
|
request: LoadRequest,
|
|
_auth: Annotated[None, Depends(require_auth)],
|
|
dep: BrowserDep,
|
|
) -> list[LoadResult]:
|
|
"""
|
|
Fetch URLs through the anti-bot browser and return their text content.
|
|
|
|
Each URL is fetched sequentially; a failing URL yields empty
|
|
page_content so Open WebUI's RAG pipeline degrades gracefully.
|
|
"""
|
|
results: list[LoadResult] = []
|
|
for url in request.urls:
|
|
try:
|
|
await dep.page.goto(url, timeout=60_000)
|
|
await dep.page.wait_for_load_state("domcontentloaded", timeout=30_000)
|
|
try:
|
|
await dep.page.wait_for_load_state("networkidle", timeout=15_000)
|
|
except PlaywrightTimeoutError:
|
|
logger.debug("networkidle timed out for %s; extracting anyway", url)
|
|
content = await _extract_content(dep.page)
|
|
except Exception as exc:
|
|
logger.warning("Failed to load %s: %s", url, exc)
|
|
content = ""
|
|
results.append(LoadResult(page_content=content, metadata={"source": url}))
|
|
return results
|