add manual build

Bump docker/metadata-action from 5.5.1 to 5.6.1 in /.github/workflows

.devcontainer/Dockerfile

@@ -1,5 +1,5 @@
-FROM python:3.12
+FROM python:3.12-bullseye
 
-RUN apt update && apt upgrade -y && apt install -y chromium xvfb
+RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb
 
 RUN curl -sSL https://install.python-poetry.org | python3 -

.devcontainer/devcontainer.json

@@ -32,6 +32,7 @@
     "8181:8191"
   ],
   "features": {
-    "ghcr.io/devcontainers-extra/features/act:1": {}
+    "ghcr.io/devcontainers-extra/features/act:1": {},
+    "ghcr.io/devcontainers/features/desktop-lite:1": {}
   }
 }

.github/ISSUE_TEMPLATE/bug_report.md

@@ -11,14 +11,17 @@ assignees: ''
 A clear and concise description of what the bug is.
 
 **I've:**
+
 - [ ] - read Loop Warning on Readme.
 - [ ] - done the troubleshooting from Readme.
 - [ ] - checked if such issue already exists.
 
 **Docker host (please complete the following information):**
- - OS: [e.g. Ubuntu Linux]
- - Platform and model: [Synology, PC, Laptop, RPi 3]
- - Architecture [e.g. ARM64]
+
+- OS: [e.g. Ubuntu Linux]
+- Platform and model: [Synology, PC, Laptop, RPi 3]
+- Architecture [e.g. ARM64]
+- Website: [e.g. https://nowsecure.nl]
 
 **Additional context**
 Add any other context about the problem here.

.github/workflows/docker-publish.yml

@@ -14,6 +14,7 @@ on:
     tags: ["v*.*.*"]
   pull_request:
     branches: ["main"]
+  workflow_dispatch:
 
 env:
   # Use docker.io for Docker Hub if empty
@@ -30,6 +31,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+          cache: "pip"
 
       - name: Set up Poetry
         run: pip install poetry
@@ -55,7 +57,13 @@ jobs:
           rm ./google-chrome-stable_current_amd64.deb
 
       - name: Run tests
-        run: poetry run pytest
+        run: poetry run pytest --retries 2 -n auto --xvfb
+
+      - name: Upload screenshots if tests fail
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          path: screenshots
 
   build:
     needs: test
@@ -99,7 +107,7 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           tags: |
             type=ref,event=branch

.vscode/launch.json

@@ -1,27 +1,19 @@
 {
+    "configurations": [
+        {
+            "console": "integratedTerminal",
+            "env": {
+                "LOG_LEVEL": "DEBUG"
+            },
+            "justMyCode": false,
+            "name": "Python Debugger: Main",
+            "program": "main.py",
+            "request": "launch",
+            "type": "debugpy"
+        },
+    ],
     // Use IntelliSense to learn about possible attributes.
     // Hover to view descriptions of existing attributes.
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Python Debugger: Main",
-            "type": "debugpy",
-            "justMyCode": false,
-            "env": {
-                "LOG_LEVEL": "DEBUG",
-            },
-            "request": "launch",
-            "program": "main.py",
-            "console": "integratedTerminal"
-        },
-        {
-            "name": "Python Debugger: Current File",
-            "justMyCode": false,
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal"
-        }
-    ]
+    "version": "0.2.0"
 }

.vscode/settings.json

@@ -1,20 +1,23 @@
 {
-    "python.testing.pytestArgs": [
-        "tests"
-    ],
-    "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true,
-    "python.analysis.autoImportCompletions": true,
-    "python.analysis.typeCheckingMode": "basic",
-    "python.terminal.activateEnvironment": true,
     "[python]": {
-        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.defaultFormatter": "charliermarsh.ruff"
     },
+    "python.analysis.autoImportCompletions": true,
     "python.analysis.packageIndexDepths": [
         {
-            "name": "",
             "depth": 5,
             "includeAllSymbols": true,
+            "name": ""
         }
     ],
+    "python.analysis.typeCheckingMode": "standard",
+    "python.terminal.activateEnvironment": true,
+    "python.testing.pytestArgs": [
+        "-n",
+        "auto",
+        "--retries=2",
+        "--xvfb"
+    ],
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false
 }

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim
+FROM python:3.12-slim-bullseye
 
 # Inspired by https://github.com/Hudrolax/uc-docker-alpine/
 
@@ -19,7 +19,7 @@ ENV \
 WORKDIR /app
 EXPOSE 8191
 RUN apt update &&\
-    apt install -y xvfb scrot python3-tk curl chromium
+    apt install -y xvfb scrot python3-tk curl chromium chromium-driver
 
 RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH="${HOME}/.local/bin:$PATH"
@@ -27,4 +27,6 @@ COPY pyproject.toml poetry.lock ./
 RUN poetry install
 
 COPY . .
-CMD [". .venv/bin/activate && python3 main.py"]
+HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
+RUN ln /usr/bin/chromedriver ./.venv/lib/python3.12/site-packages/seleniumbase/drivers/uc_driver
+CMD ["./cmd.sh"]

README.md

@@ -1,36 +1,27 @@
 # Byparr
 
-An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) as a drop-in replacement, build with [nodriver](https://github.com/ultrafunkamsterdam/nodriver) and [FastAPI](https://fastapi.tiangolo.com).
+An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) as a drop-in replacement, build with [seleniumbase](https://seleniumbase.io/) and [FastAPI](https://fastapi.tiangolo.com).
 
 > [!WARNING]
-> Due to recent challenge changes, this software does not guarantee that the Cloudflare challenge will be bypassed. Cloudflare likely requires valid network traffic originating from the user’s public IP address to mark a connection as legitimate. While this tool may bypass the initial browser check, it does not ensure that requests will consistently pass Cloudflare's validation. More testing and data are required to understand how Cloudflare identifies connections and requests as valid. Invalid requests will result in Byparr's looping and eventually timeouting.
+> Due to recent challenge changes, this software does not guarantee that the Cloudflare challenge will be bypassed. Cloudflare likely requires valid network traffic originating from the user’s public IP address to mark a connection as legitimate. While this tool may bypass the initial browser check, it does not ensure that requests will consistently pass Cloudflare's validation. More testing and data are required to understand how Cloudflare identifies connections and requests as valid. Invalid requests will result in Byparr's looping and eventually time-outing.
 
-> [!IMPORTANT]
-> Currenly, due to [bug in nodriver](https://github.com/ultrafunkamsterdam/undetected-chromedriver/issues/1954), if you want to run this project ouside of prebuild container, you have to run `python fix_nodriver.py` after creating venv to patch the library.
+> [!WARNING]
+> Support for NAS devices (like Synology) is minimal. Please report issues, but do not expect it to be fixed quickly. The only ARM device I have is a free Ampere Oracle VM, so I can only test ARM support on that.
 
 > [!NOTE]
 > Thanks to FastAPI implementation, now you can also see the API documentation at `/docs` or `/` (redirect to `/docs`) endpoints.
 
-## Early Development
-
-Long story short, I created it in like 3 days, so if you get any bugs/hangs etc. please report it, so both of us can enjoy unblocked websites!
-
-I focus maily on Cloudflare, which is tested daily, any other anti-bot challenges should pass out of the box, but if any issues, please report these providers with an example website ❤️
-
 ## Troubleshooting
 
 1. Clone repo to the host that has the container has issues on.
 2. Using vscode and `SSH extention`, connect to the host and open repo in it.
-4. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
-3. Open vscode terminal (`` CTRL + ` ``) and run `./run_vnc.sh` script.
-5. Write down the port from `Forwarded Address` in `Ports` tab in vscode (probably 5900).
-6. Open <https://novnc.com/noVNC/vnc.html> in your pc's browser and using settings on left under websocket, set host to `localhost` nad port to the port you wrote down and disable encryption.
-7. Connect to the VNC server on container.
-8. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`
-9. If chromium works, run (or run debug) tests from VS Code.
-10. Check if everything works by observing VNC in the browser as code is being tested.
-11. If code works, congrats! (/s) You are on your own.
-12. If it does not, try another host or network and try again.
+3. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
+4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
+5. Open `http://localhost:6080` and connect to the virtual desktop.
+6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
+7. If chromium works, run (or debug) tests from VS Code.
+   1. If code works, congrats! (/s) You are on your own.
+   2. If it does not, try another host or network, try again and create issue about the problem.
 
 ## Usage
 
@@ -39,18 +30,18 @@ I focus maily on Cloudflare, which is tested daily, any other anti-bot challenge
 ```yaml
 services:
   byparr:
-    image: ghcr.io/thephaseless/byparr
+    image: ghcr.io/thephaseless/byparr:latest
     environment:
       - LOG_LEVEL=INFO # optional
     ports:
-      - "8191:8191" # Optional if used with *arr network
+      - "8191:8191" # Optional if needed to make make requests/check docs on host
 ```
 
 ## Need help with / TODO
 
-- [ ] Slimming container (~3GB bruh)
-- [ ] Add more anti-bot challenges
+- [x] Slimming container (only ~650 mB now!)
+- [x] Add more anti-bot challenges
 - [x] Add docstrings
-- [ ] Implement versioning
+- [x] Implement versioning
 - [ ] Proxy support
-- [ ] Add more architectures support
+- [x] Add more architectures support

cmd.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+. .venv/bin/activate && python3 main.py

fix_nodriver.py

@@ -1,65 +0,0 @@
-# https://github.com/ultrafunkamsterdam/undetected-chromedriver/issues/1954
-# Fix for nodriver in .venv/lib/python3.11/site-packages/nodriver/core/browser.py
-from __future__ import annotations
-
-import logging
-import os
-from pathlib import Path
-from platform import python_version
-
-env_path = os.getenv("VIRTUAL_ENV")
-if env_path is None:
-    env_path = Path(os.__file__).parent.parent.parent.as_posix()
-python_version = python_version().split(".")[0:2]
-nodriver_path = Path(env_path + f"/lib/python{'.'.join(python_version)}/site-packages/nodriver/cdp/network.py")
-if not nodriver_path.exists():
-    msg = f"{nodriver_path} not found"
-    raise FileNotFoundError(msg)
-
-new_cookie_partition_key = """\
-        if isinstance(json, str):
-            return cls(top_level_site=json, has_cross_site_ancestor=False)
-        elif isinstance(json, dict):
-            return cls(
-                top_level_site=str(json["topLevelSite"]),
-                has_cross_site_ancestor=bool(json["hasCrossSiteAncestor"]),
-            )
-"""
-
-logger = logging.getLogger(__name__)
-handler = logging.StreamHandler()
-logger.addHandler(handler)
-logger.setLevel(logging.INFO)
-logger.info(f"Fixing nodriver in {nodriver_path}")
-# delete CookiePartitionKey declaration
-with nodriver_path.open("r+") as f:
-    lines = f.readlines()
-    found_def = False
-    found_body = False
-    i = -1
-    while i < len(lines):
-        i += 1
-        line = lines[i]
-        strip_line = line.strip("\n")
-        if not found_def and line.startswith("class CookiePartitionKey:"):
-            logger.info(f"Found line {i}: {strip_line}")
-            found_def = True
-            continue
-        if found_def:
-            if line.startswith("    def from_json"):
-                logger.info(f"Found line {i}: {strip_line}")
-                found_body = True
-                continue
-            if found_body:
-                if line.startswith(("\t\t", "        ")):
-                    logger.info(f"Removing line {i}: {strip_line}")
-                    lines.pop(i)
-                    i -= 1
-                    continue
-                else:
-                    lines = lines[:i] + [new_cookie_partition_key] + lines[i:]
-                    break
-
-
-with nodriver_path.open("w") as f:
-    f.writelines(lines)

main.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import time
+from http import HTTPStatus
 
 import uvicorn.config
 from bs4 import BeautifulSoup
@@ -30,9 +31,16 @@ def read_root():
 async def health_check():
     """Health check endpoint."""
     logger.info("Health check")
-    # browser: Chrome = await new_browser()
-    # browser.get("https://google.com")
-    # browser.stop()
+
+    health_check_request = read_item(
+        LinkRequest.model_construct(url="https://prowlarr.servarr.com/v1/ping")
+    )
+    if health_check_request.solution.status != HTTPStatus.OK:
+        raise HTTPException(
+            status_code=500,
+            detail="Health check failed",
+        )
+
     return {"status": "ok"}
 
 
@@ -45,37 +53,44 @@ def read_item(request: LinkRequest):
     response: LinkResponse
 
     # start_time = int(time.time() * 1000)
-    with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
-        sb: BaseCase
-        sb.uc_open_with_reconnect(request.url)
-        sb.uc_gui_click_captcha()
-        logger.info(f"Got webpage: {request.url}")
-        sb.save_screenshot("screenshot.png")
-        logger.info(f"Got webpage: {request.url}")
+    try:
+        with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
+            sb: BaseCase
+            sb.uc_open_with_reconnect(request.url)
+            source = sb.get_page_source()
+            source_bs = BeautifulSoup(source, "html.parser")
+            title_tag = source_bs.title
+            logger.info(f"Got webpage: {request.url}")
+            if title_tag and title_tag.string in src.utils.consts.CHALLENGE_TITLES:
+                logger.info("Challenge detected")
+                sb.uc_gui_click_captcha()
+                logger.info("Clicked captcha")
 
-        source = sb.get_page_source()
-        source_bs = BeautifulSoup(source, "html.parser")
-        title_tag = source_bs.title
-        if title_tag is None:
-            raise HTTPException(status_code=500, detail="Title tag not found")
+            source = sb.get_page_source()
+            source_bs = BeautifulSoup(source, "html.parser")
+            title_tag = source_bs.title
 
-        if title_tag.string in src.utils.consts.CHALLENGE_TITLES:
-            raise HTTPException(status_code=500, detail="Could not bypass challenge")
+            if title_tag and title_tag.string in src.utils.consts.CHALLENGE_TITLES:
+                sb.save_screenshot(f"./screenshots/{request.url}.png")
+                raise HTTPException(
+                    status_code=500, detail="Could not bypass challenge"
+                )
 
-        title = title_tag.string
-        logger.info(f"Title: {title}")
-        response = LinkResponse(
-            message="Success",
-            solution=Solution(
-                userAgent=sb.get_user_agent(),
-                url=sb.get_current_url(),
-                status=200,
-                cookies=sb.get_cookies(),
-                headers={},
-                response=source,
-            ),
-            startTimestamp=start_time,
-        )
+            response = LinkResponse(
+                message="Success",
+                solution=Solution(
+                    userAgent=sb.get_user_agent(),
+                    url=sb.get_current_url(),
+                    status=200,
+                    cookies=sb.get_cookies(),
+                    headers={},
+                    response=source,
+                ),
+                startTimestamp=start_time,
+            )
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        raise HTTPException(status_code=500, detail="Unknown error, check logs") from e
 
     return response
 

poetry.lock

@@ -1324,6 +1324,19 @@ files = [
 [package.dependencies]
 pytest = "*"
 
+[[package]]
+name = "pytest-progress"
+version = "1.3.0"
+description = "pytest plugin for instant test progress status"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pytest_progress-1.3.0.tar.gz", hash = "sha256:b2a6cd0b0cd8b50b19f56777402835e546dc8404eb7fa77ac2ace9dc719ec50e"},
+]
+
+[package.dependencies]
+pytest = ">=2.7"
+
 [[package]]
 name = "pytest-rerunfailures"
 version = "14.0"
@@ -1339,6 +1352,23 @@ files = [
 packaging = ">=17.1"
 pytest = ">=7.2"
 
+[[package]]
+name = "pytest-retry"
+version = "1.6.3"
+description = "Adds the ability to retry flaky tests in CI environments"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "pytest_retry-1.6.3-py3-none-any.whl", hash = "sha256:e96f7df77ee70b0838d1085f9c3b8b5b7d74bf8947a0baf32e2b8c71b27683c8"},
+    {file = "pytest_retry-1.6.3.tar.gz", hash = "sha256:36ccfa11c8c8f9ddad5e20375182146d040c20c4a791745139c5a99ddf1b557d"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0"
+
+[package.extras]
+dev = ["black", "flake8", "isort", "mypy"]
+
 [[package]]
 name = "pytest-xdist"
 version = "3.6.1"
@@ -2165,4 +2195,4 @@ h11 = ">=0.9.0,<1"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "1dcc6c3a9ff83a4e27c96b1047a388e1ddd9a7c326b20ef07793c0721f9594dc"
+content-hash = "531739092d03e663fe62d41b76d59b38204657717413e297158b6a21779e6075"

pyproject.toml

@@ -17,6 +17,9 @@ ruff = "^0.8.0"
 seleniumbase = "^4.32.12"
 pyautogui = "^0.9.54"
 beautifulsoup4 = "^4.12.3"
+pytest-retry = "^1.6.3"
+pytest-progress = "^1.3.0"
+pytest-xdist = "^3.6.1"
 
 
 [build-system]

src/models/requests.py

@@ -4,13 +4,13 @@ import time
 from http import HTTPStatus
 from typing import Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 class LinkRequest(BaseModel):
-    cmd: str
+    cmd: str = "get"
     url: str
-    maxTimeout: int  # noqa: N815 # Ignore to preserve compatibility
+    max_timeout: int = Field(30, alias="maxTimeout")
 
 
 class ProtectionTriggeredError(Exception):

tests/main_test.py

@@ -39,7 +39,9 @@ def test_bypass(website: str):
 
     response = client.post(
         "/v1",
-        json=LinkRequest(url=website, maxTimeout=30, cmd="request.get").model_dump(),
+        json=LinkRequest.model_construct(
+            url=website, max_timeout=30, cmd="request.get"
+        ).model_dump(),
     )
 
     assert response.status_code == HTTPStatus.OK