mirror of
https://github.com/ThePhaseless/Byparr.git
synced 2026-09-24 22:03:30 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c37ab51179 | ||
|
|
962db9409c | ||
|
|
96a9cc7ba9 | ||
|
|
bf9460b960 | ||
|
|
c0547192dd | ||
|
|
008077076c | ||
|
|
b33444dc48 | ||
|
|
b8c8576e5a | ||
|
|
905d802a26 | ||
|
|
f99429fdea | ||
|
|
1a4ec03587 | ||
|
|
ee2630dce2 | ||
|
|
82130b77c0 | ||
|
|
faf7e5e51a | ||
|
|
597faa48dc | ||
|
|
a28dbbe972 | ||
|
|
82124dee9b | ||
|
|
0551c7b82b | ||
|
|
b483dba080 | ||
|
|
7533d039ac | ||
|
|
38b2ea0798 | ||
|
|
58ae7964df | ||
|
|
5283fb2d0e | ||
|
|
b3e7662782 | ||
|
|
074a3dd712 | ||
|
|
cb48f16370 | ||
|
|
637de7572d | ||
|
|
7b55252a73 | ||
|
|
cbe4cf57a5 | ||
|
|
0775d107ad | ||
|
|
d36972b80e | ||
|
|
27ec17f7f2 | ||
|
|
0b2bf9e12a | ||
|
|
8c6906c916 | ||
|
|
3103051ccf |
@@ -1,5 +1,6 @@
|
||||
FROM python:3.12-bullseye
|
||||
FROM debian:latest
|
||||
|
||||
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb
|
||||
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb git
|
||||
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
@@ -23,50 +23,7 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Set up Poetry
|
||||
run: pip install poetry
|
||||
|
||||
- name: Setup a local virtual environment (if no poetry.toml file)
|
||||
run: |
|
||||
poetry config virtualenvs.create true --local
|
||||
poetry config virtualenvs.in-project true --local
|
||||
|
||||
- uses: actions/cache@v4
|
||||
name: Define a cache for the virtual environment based on the dependencies lock file
|
||||
with:
|
||||
path: ./.venv
|
||||
key: venv-${{ hashFiles('poetry.lock') }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
poetry install
|
||||
sudo apt update
|
||||
sudo apt install -y xvfb scrot python3-tk
|
||||
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
|
||||
sudo apt install -y ./google-chrome-stable_current_amd64.deb
|
||||
rm ./google-chrome-stable_current_amd64.deb
|
||||
|
||||
- name: Run tests
|
||||
run: poetry run pytest --retries 2 -n auto --xvfb
|
||||
|
||||
- name: Upload screenshots if tests fail
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
path: screenshots
|
||||
|
||||
build:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -83,21 +40,19 @@ jobs:
|
||||
# https://github.com/sigstore/cosign-installer
|
||||
- name: Install cosign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da #v3.7.0
|
||||
with:
|
||||
cosign-release: "v2.2.4"
|
||||
uses: sigstore/cosign-installer@v3
|
||||
|
||||
# Set up BuildKit Docker container builder to be able to build
|
||||
# multi-platform images and export cache
|
||||
# https://github.com/docker/setup-buildx-action
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into registry ${{ env.REGISTRY }}
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -107,7 +62,7 @@ jobs:
|
||||
# https://github.com/docker/metadata-action
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
@@ -118,20 +73,32 @@ jobs:
|
||||
type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
|
||||
- name: Test
|
||||
id: test
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha
|
||||
pull: true
|
||||
cache-to: type=gha,mode=max
|
||||
target: test
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
pull: true
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: GITHUB_BUILD=true
|
||||
build-args: GITHUB_BUILD=true,VERSION=${{ github.ref_name }}
|
||||
|
||||
# Sign the resulting Docker image digest except on PRs.
|
||||
# This will only write to the public Rekor transparency log when the Docker
|
||||
|
||||
+17
-12
@@ -1,4 +1,4 @@
|
||||
FROM python:3.12-slim-bullseye
|
||||
FROM debian:bullseye-slim AS base
|
||||
|
||||
# Inspired by https://github.com/Hudrolax/uc-docker-alpine/
|
||||
|
||||
@@ -11,22 +11,27 @@ ENV \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
# prevents python creating .pyc files
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
# do not ask any interactive question
|
||||
POETRY_NO_INTERACTION=1 \
|
||||
POETRY_VIRTUALENVS_IN_PROJECT=true \
|
||||
PATH="${HOME}/.local/bin:$PATH" \
|
||||
DISPLAY=:0
|
||||
|
||||
WORKDIR /app
|
||||
EXPOSE 8191
|
||||
RUN apt update &&\
|
||||
apt install -y xvfb scrot python3-tk curl chromium chromium-driver
|
||||
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
ENV PATH="${HOME}/.local/bin:$PATH"
|
||||
COPY pyproject.toml poetry.lock ./
|
||||
RUN poetry install
|
||||
apt upgrade -y &&\
|
||||
apt install -y --no-install-recommends --no-install-suggests xauth xvfb scrot python3-tk curl chromium chromium-driver ca-certificates x11-common
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv sync
|
||||
|
||||
COPY . .
|
||||
RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver
|
||||
|
||||
FROM base AS test
|
||||
|
||||
RUN uv sync --group test
|
||||
RUN ./test.sh
|
||||
|
||||
FROM base
|
||||
|
||||
EXPOSE 8191
|
||||
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
|
||||
RUN ln /usr/bin/chromedriver ./.venv/lib/python3.12/site-packages/seleniumbase/drivers/uc_driver
|
||||
CMD ["./cmd.sh"]
|
||||
@@ -14,14 +14,14 @@ An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) a
|
||||
## Troubleshooting
|
||||
|
||||
1. Clone repo to the host that has the container has issues on.
|
||||
2. Using vscode and `SSH extention`, connect to the host and open repo in it.
|
||||
3. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
|
||||
2. Using vscode and `SSH extension`, connect to the host and open repo in it.
|
||||
3. Download `Dev Containers` extension and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
|
||||
4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
|
||||
5. Open `http://localhost:6080` and connect to the virtual desktop.
|
||||
6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
|
||||
7. If chromium works, run (or debug) tests from VS Code.
|
||||
1. If code works, congrats! (/s) You are on your own.
|
||||
2. If it does not, try another host or network, try again and create issue about the problem.
|
||||
1. If code works, congrats! (not really) You are on your own.
|
||||
2. If it does not, try on another host or network and create an issue if problem persists.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -37,11 +37,23 @@ services:
|
||||
- "8191:8191" # Optional if needed to make make requests/check docs on host
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
|
||||
```
|
||||
|
||||
### Local
|
||||
|
||||
```bash
|
||||
uv sync && ./cmd.sh
|
||||
```
|
||||
|
||||
## Need help with / TODO
|
||||
|
||||
- [x] Slimming container (only ~650 mB now!)
|
||||
- [x] Slimming container (only ~650 MB now!)
|
||||
- [x] Add more anti-bot challenges
|
||||
- [x] Add docstrings
|
||||
- [x] Add doc strings
|
||||
- [x] Implement versioning
|
||||
- [ ] Proxy support
|
||||
- [x] Add more architectures support
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
. .venv/bin/activate && python3 main.py
|
||||
uv run main.py
|
||||
@@ -45,7 +45,7 @@ async def health_check():
|
||||
|
||||
|
||||
@app.post("/v1")
|
||||
def read_item(request: LinkRequest):
|
||||
def read_item(request: LinkRequest) -> LinkResponse:
|
||||
"""Handle POST requests."""
|
||||
start_time = int(time.time() * 1000)
|
||||
# request.url = "https://nowsecure.nl"
|
||||
@@ -53,8 +53,8 @@ def read_item(request: LinkRequest):
|
||||
response: LinkResponse
|
||||
|
||||
# start_time = int(time.time() * 1000)
|
||||
try:
|
||||
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
|
||||
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
|
||||
try:
|
||||
sb: BaseCase
|
||||
sb.uc_open_with_reconnect(request.url)
|
||||
source = sb.get_page_source()
|
||||
@@ -86,9 +86,13 @@ def read_item(request: LinkRequest):
|
||||
),
|
||||
startTimestamp=start_time,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Unknown error, check logs") from e
|
||||
except Exception as e:
|
||||
logger.error(f"Error: {e}")
|
||||
if sb.driver:
|
||||
sb.driver.quit()
|
||||
raise HTTPException(
|
||||
status_code=500, detail="Unknown error, check logs"
|
||||
) from e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
Generated
-2239
File diff suppressed because it is too large
Load Diff
+24
-30
@@ -1,38 +1,32 @@
|
||||
[tool.poetry]
|
||||
# cspell:disable
|
||||
[project]
|
||||
authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
|
||||
license = { text = "LICENSE" }
|
||||
requires-python = "<4.0,>=3.12"
|
||||
name = "Byparr"
|
||||
version = "0.1.0"
|
||||
description = "API for getting cookies for Cloudflare challenges"
|
||||
package-mode = false
|
||||
authors = ["ThePhaseless <kukubaorch@gmail.com>"]
|
||||
readme = "README.md"
|
||||
license = "LICENSE"
|
||||
repository = "https://github.com/ThePhaseless/Byparr"
|
||||
dependencies = [
|
||||
"beautifulsoup4>=4.12.3",
|
||||
"fastapi[standard]>=0.115.6",
|
||||
"pyautogui>=0.9.54",
|
||||
"pydantic>=2.10.3",
|
||||
"seleniumbase>=4.33.7",
|
||||
"uvicorn>=0.32.1",
|
||||
]
|
||||
urls = { repository = "https://github.com/ThePhaseless/Byparr" }
|
||||
|
||||
# cspell:disable
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.12"
|
||||
fastapi = { extras = ["standard"], version = "^0" }
|
||||
seleniumbase = "^4.33.1"
|
||||
beautifulsoup4 = "^4.12.3"
|
||||
uvicorn = "^0.32.1"
|
||||
pydantic = "^2.10.2"
|
||||
pyautogui = "^0.9.54"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
ruff = "^0.8.0"
|
||||
deptry = "^0.21.1"
|
||||
|
||||
[tool.poetry.group.test.dependencies]
|
||||
pytest = "^8.3.3"
|
||||
httpx = "^0.27.2"
|
||||
pytest-retry = "^1.6.3"
|
||||
pytest-progress = "^1.3.0"
|
||||
pytest-asyncio = "^0"
|
||||
pytest-xdist = "^3.6.1"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
[dependency-groups]
|
||||
test = [
|
||||
"httpx>=0.28.1",
|
||||
"pytest>=8.3.4",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"pytest-progress>=1.3.0",
|
||||
"pytest-retry>=1.6.3",
|
||||
"pytest-xdist>=3.6.1",
|
||||
]
|
||||
dev = ["deptry>=0.21.1", "ruff>=0.8.2"]
|
||||
|
||||
[tool.deptry.per_rule_ignores]
|
||||
DEP002 = ["pyautogui"]
|
||||
|
||||
+18
-7
@@ -6,6 +6,8 @@ from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.utils import consts
|
||||
|
||||
|
||||
class LinkRequest(BaseModel):
|
||||
cmd: str = "get"
|
||||
@@ -26,9 +28,14 @@ class Solution(BaseModel):
|
||||
response: str
|
||||
|
||||
@classmethod
|
||||
def empty(cls):
|
||||
def invalid(cls, url: str):
|
||||
"""
|
||||
Return an empty Solution with default values.
|
||||
|
||||
Useful for returning an error response.
|
||||
"""
|
||||
return cls(
|
||||
url="",
|
||||
url=url,
|
||||
status=HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
cookies=[],
|
||||
userAgent="",
|
||||
@@ -43,19 +50,23 @@ class LinkResponse(BaseModel):
|
||||
solution: Solution
|
||||
startTimestamp: int # noqa: N815 # Ignore to preserve compatibility
|
||||
endTimestamp: int = int(time.time() * 1000) # noqa: N815 # Ignore to preserve compatibility
|
||||
version: str = "3.3.21" # TODO: Implement versioning
|
||||
version: str = consts.VERSION
|
||||
|
||||
@classmethod
|
||||
def invalid(cls):
|
||||
def invalid(cls, url: str):
|
||||
"""
|
||||
Return an invalid LinkResponse with default error values.
|
||||
|
||||
This method is used to generate a response indicating an invalid request.
|
||||
"""
|
||||
return cls(
|
||||
status="error",
|
||||
message="Invalid request",
|
||||
solution=Solution.empty(),
|
||||
solution=Solution.invalid(url),
|
||||
startTimestamp=int(time.time() * 1000),
|
||||
endTimestamp=int(time.time() * 1000),
|
||||
version="3.3.21",
|
||||
)
|
||||
|
||||
|
||||
class NoChromeExtensionError(Exception):
|
||||
"""No chrome extention found."""
|
||||
"""No chrome extension found."""
|
||||
|
||||
@@ -1,9 +1,33 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
def get_version_from_env():
|
||||
"""
|
||||
Retrieve the version from the environment variable 'VERSION'.
|
||||
|
||||
This function checks the 'VERSION' environment variable for a value
|
||||
that starts with 'v' and returns the version without the prefix.
|
||||
|
||||
Returns:
|
||||
str | None: The version string without the 'v' prefix, or None if
|
||||
the 'VERSION' environment variable is not set or does not start
|
||||
with 'v'.
|
||||
|
||||
"""
|
||||
version_env = os.getenv("VERSION")
|
||||
if not version_env or not version_env.startswith("v"):
|
||||
return None
|
||||
|
||||
return version_env.removeprefix("v")
|
||||
|
||||
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL") or "INFO"
|
||||
LOG_LEVEL = logging.getLevelNamesMapping()[LOG_LEVEL.upper()]
|
||||
|
||||
VERSION = get_version_from_env() or "unknown"
|
||||
|
||||
|
||||
CHALLENGE_TITLES = [
|
||||
# Cloudflare
|
||||
"Just a moment...",
|
||||
|
||||
@@ -12,19 +12,12 @@ client = TestClient(app)
|
||||
test_websites = [
|
||||
"https://btmet.com/",
|
||||
"https://ext.to/",
|
||||
]
|
||||
|
||||
# Not testable on github actions
|
||||
github_restricted = [
|
||||
"https://www.ygg.re/",
|
||||
"https://extratorrent.st/",
|
||||
"https://idope.se/",
|
||||
"https://speed.cd/login",
|
||||
]
|
||||
|
||||
# if os.getenv("GITHUB_ACTIONS") != "true":
|
||||
test_websites.extend(github_restricted)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("website", test_websites)
|
||||
def test_bypass(website: str):
|
||||
|
||||
Reference in New Issue
Block a user