Compare commits

..
35 Commits
Author SHA1 Message Date
ThePhaseless c37ab51179 combine apt installs to one 2024-12-11 01:04:08 +00:00
ThePhaseless 962db9409c pull before build and test 2024-12-11 01:02:59 +00:00
ThePhaseless 96a9cc7ba9 test, then build 2024-12-11 01:01:00 +00:00
ThePhaseless bf9460b960 delete unused file 2024-12-11 00:52:20 +00:00
ThePhaseless c0547192dd test only on amd64 2024-12-11 00:46:53 +00:00
ThePhaseless 008077076c try using arm builder 2024-12-11 00:44:22 +00:00
ThePhaseless b33444dc48 use matrix for build 2024-12-11 00:40:11 +00:00
ThePhaseless b8c8576e5a github actions shared lib fix 2024-12-11 00:21:07 +00:00
ThePhaseless 905d802a26 add back retires 2024-12-10 23:38:13 +00:00
ThePhaseless f99429fdea update all packages but install only needed 2024-12-10 23:28:29 +00:00
ThePhaseless 1a4ec03587 Merge pull request #37 from ThePhaseless/uv
Uv
2024-12-10 23:50:34 +01:00
ThePhaseless ee2630dce2 enable auto update 2024-12-10 22:48:53 +00:00
ThePhaseless 82130b77c0 fix docker and update readme 2024-12-10 22:48:35 +00:00
ThePhaseless faf7e5e51a migrate to uv 2024-12-10 22:16:30 +00:00
ThePhaseless 597faa48dc add response type 2024-12-10 21:54:39 +00:00
ThePhaseless a28dbbe972 use docker to test 2024-12-10 21:54:29 +00:00
ThePhaseless 82124dee9b force exit on exception 2024-12-10 21:29:20 +00:00
ThePhaseless 0551c7b82b use uv 2024-12-08 16:49:38 +00:00
ThePhaseless b483dba080 [skip ci] Mili to Mega 2024-12-02 10:29:13 +01:00
ThePhaseless 7533d039ac Merge pull request #30 from ThePhaseless/renovate/python-3.x
Update python Docker tag to v3.13
2024-11-30 14:26:22 +01:00
renovate[bot] 38b2ea0798 Update python Docker tag to v3.13 2024-11-30 12:59:43 +00:00
ThePhaseless 58ae7964df remove python action update script 2024-11-30 12:59:00 +00:00
ThePhaseless 5283fb2d0e exclude python from autoupdate 2024-11-30 12:58:02 +00:00
ThePhaseless b3e7662782 fix ruff 2024-11-30 12:48:04 +00:00
ThePhaseless 074a3dd712 [skip ci] fix renovate 2024-11-30 12:47:34 +00:00
ThePhaseless cb48f16370 update python version in action with base 2024-11-30 12:45:19 +00:00
ThePhaseless 637de7572d use latest cosign 2024-11-30 12:44:08 +00:00
ThePhaseless 7b55252a73 run ruff before tests 2024-11-30 12:43:56 +00:00
ThePhaseless cbe4cf57a5 use all urls always 2024-11-30 12:43:05 +00:00
ThePhaseless 0775d107ad add version control and more verbose error return 2024-11-30 12:42:51 +00:00
ThePhaseless d36972b80e update python in action 2024-11-30 12:42:17 +00:00
renovate[bot] 27ec17f7f2 Update dependency seleniumbase to v4.33.2 (#24)
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2024-11-30 12:08:29 +00:00
ThePhaseless 0b2bf9e12a use dynamic driver symlink 2024-11-30 11:38:48 +00:00
renovate[bot] 8c6906c916 Update dependency ruff to v0.8.1 (#29)
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2024-11-29 06:26:30 +00:00
renovate[bot] 3103051ccf Update dependency httpx to ^0.28.0 (#28)
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2024-11-28 18:20:06 +00:00
13 changed files with 1587 additions and 2362 deletions
+4 -3
View File
@@ -1,5 +1,6 @@
FROM python:3.12-bullseye
FROM debian:latest
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb git
RUN curl -sSL https://install.python-poetry.org | python3 -
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
ENV PATH="/root/.local/bin:$PATH"
+18 -51
View File
@@ -23,50 +23,7 @@ env:
IMAGE_NAME: ${{ github.repository }}
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
- name: Set up Poetry
run: pip install poetry
- name: Setup a local virtual environment (if no poetry.toml file)
run: |
poetry config virtualenvs.create true --local
poetry config virtualenvs.in-project true --local
- uses: actions/cache@v4
name: Define a cache for the virtual environment based on the dependencies lock file
with:
path: ./.venv
key: venv-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
run: |
poetry install
sudo apt update
sudo apt install -y xvfb scrot python3-tk
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo apt install -y ./google-chrome-stable_current_amd64.deb
rm ./google-chrome-stable_current_amd64.deb
- name: Run tests
run: poetry run pytest --retries 2 -n auto --xvfb
- name: Upload screenshots if tests fail
if: failure()
uses: actions/upload-artifact@v4
with:
path: screenshots
build:
needs: test
runs-on: ubuntu-latest
permissions:
contents: read
@@ -83,21 +40,19 @@ jobs:
# https://github.com/sigstore/cosign-installer
- name: Install cosign
if: github.event_name != 'pull_request'
uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da #v3.7.0
with:
cosign-release: "v2.2.4"
uses: sigstore/cosign-installer@v3
# Set up BuildKit Docker container builder to be able to build
# multi-platform images and export cache
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
uses: docker/setup-buildx-action@v3
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -107,7 +62,7 @@ jobs:
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
uses: docker/metadata-action@v5
with:
tags: |
type=ref,event=branch
@@ -118,20 +73,32 @@ jobs:
type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Test
id: test
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64
cache-from: type=gha
pull: true
cache-to: type=gha,mode=max
target: test
# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0
uses: docker/build-push-action@v6
with:
context: .
pull: true
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: GITHUB_BUILD=true
build-args: GITHUB_BUILD=true,VERSION=${{ github.ref_name }}
# Sign the resulting Docker image digest except on PRs.
# This will only write to the public Rekor transparency log when the Docker
+17 -12
View File
@@ -1,4 +1,4 @@
FROM python:3.12-slim-bullseye
FROM debian:bullseye-slim AS base
# Inspired by https://github.com/Hudrolax/uc-docker-alpine/
@@ -11,22 +11,27 @@ ENV \
PYTHONUNBUFFERED=1 \
# prevents python creating .pyc files
PYTHONDONTWRITEBYTECODE=1 \
# do not ask any interactive question
POETRY_NO_INTERACTION=1 \
POETRY_VIRTUALENVS_IN_PROJECT=true \
PATH="${HOME}/.local/bin:$PATH" \
DISPLAY=:0
WORKDIR /app
EXPOSE 8191
RUN apt update &&\
apt install -y xvfb scrot python3-tk curl chromium chromium-driver
RUN curl -sSL https://install.python-poetry.org | python3 -
ENV PATH="${HOME}/.local/bin:$PATH"
COPY pyproject.toml poetry.lock ./
RUN poetry install
apt upgrade -y &&\
apt install -y --no-install-recommends --no-install-suggests xauth xvfb scrot python3-tk curl chromium chromium-driver ca-certificates x11-common
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
COPY pyproject.toml uv.lock ./
RUN uv sync
COPY . .
RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver
FROM base AS test
RUN uv sync --group test
RUN ./test.sh
FROM base
EXPOSE 8191
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
RUN ln /usr/bin/chromedriver ./.venv/lib/python3.12/site-packages/seleniumbase/drivers/uc_driver
CMD ["./cmd.sh"]
+18 -6
View File
@@ -14,14 +14,14 @@ An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) a
## Troubleshooting
1. Clone repo to the host that has the container has issues on.
2. Using vscode and `SSH extention`, connect to the host and open repo in it.
3. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
2. Using vscode and `SSH extension`, connect to the host and open repo in it.
3. Download `Dev Containers` extension and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
5. Open `http://localhost:6080` and connect to the virtual desktop.
6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
7. If chromium works, run (or debug) tests from VS Code.
1. If code works, congrats! (/s) You are on your own.
2. If it does not, try another host or network, try again and create issue about the problem.
1. If code works, congrats! (not really) You are on your own.
2. If it does not, try on another host or network and create an issue if problem persists.
## Usage
@@ -37,11 +37,23 @@ services:
- "8191:8191" # Optional if needed to make make requests/check docs on host
```
### Docker
```bash
docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
```
### Local
```bash
uv sync && ./cmd.sh
```
## Need help with / TODO
- [x] Slimming container (only ~650 mB now!)
- [x] Slimming container (only ~650 MB now!)
- [x] Add more anti-bot challenges
- [x] Add docstrings
- [x] Add doc strings
- [x] Implement versioning
- [ ] Proxy support
- [x] Add more architectures support
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/sh
. .venv/bin/activate && python3 main.py
uv run main.py
+10 -6
View File
@@ -45,7 +45,7 @@ async def health_check():
@app.post("/v1")
def read_item(request: LinkRequest):
def read_item(request: LinkRequest) -> LinkResponse:
"""Handle POST requests."""
start_time = int(time.time() * 1000)
# request.url = "https://nowsecure.nl"
@@ -53,8 +53,8 @@ def read_item(request: LinkRequest):
response: LinkResponse
# start_time = int(time.time() * 1000)
try:
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
try:
sb: BaseCase
sb.uc_open_with_reconnect(request.url)
source = sb.get_page_source()
@@ -86,9 +86,13 @@ def read_item(request: LinkRequest):
),
startTimestamp=start_time,
)
except Exception as e:
logger.error(f"Error: {e}")
raise HTTPException(status_code=500, detail="Unknown error, check logs") from e
except Exception as e:
logger.error(f"Error: {e}")
if sb.driver:
sb.driver.quit()
raise HTTPException(
status_code=500, detail="Unknown error, check logs"
) from e
return response
Generated
-2239
View File
File diff suppressed because it is too large Load Diff
+24 -30
View File
@@ -1,38 +1,32 @@
[tool.poetry]
# cspell:disable
[project]
authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
license = { text = "LICENSE" }
requires-python = "<4.0,>=3.12"
name = "Byparr"
version = "0.1.0"
description = "API for getting cookies for Cloudflare challenges"
package-mode = false
authors = ["ThePhaseless <kukubaorch@gmail.com>"]
readme = "README.md"
license = "LICENSE"
repository = "https://github.com/ThePhaseless/Byparr"
dependencies = [
"beautifulsoup4>=4.12.3",
"fastapi[standard]>=0.115.6",
"pyautogui>=0.9.54",
"pydantic>=2.10.3",
"seleniumbase>=4.33.7",
"uvicorn>=0.32.1",
]
urls = { repository = "https://github.com/ThePhaseless/Byparr" }
# cspell:disable
[tool.poetry.dependencies]
python = "^3.12"
fastapi = { extras = ["standard"], version = "^0" }
seleniumbase = "^4.33.1"
beautifulsoup4 = "^4.12.3"
uvicorn = "^0.32.1"
pydantic = "^2.10.2"
pyautogui = "^0.9.54"
[tool.poetry.group.dev.dependencies]
ruff = "^0.8.0"
deptry = "^0.21.1"
[tool.poetry.group.test.dependencies]
pytest = "^8.3.3"
httpx = "^0.27.2"
pytest-retry = "^1.6.3"
pytest-progress = "^1.3.0"
pytest-asyncio = "^0"
pytest-xdist = "^3.6.1"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[dependency-groups]
test = [
"httpx>=0.28.1",
"pytest>=8.3.4",
"pytest-asyncio>=0.24.0",
"pytest-progress>=1.3.0",
"pytest-retry>=1.6.3",
"pytest-xdist>=3.6.1",
]
dev = ["deptry>=0.21.1", "ruff>=0.8.2"]
[tool.deptry.per_rule_ignores]
DEP002 = ["pyautogui"]
+18 -7
View File
@@ -6,6 +6,8 @@ from typing import Any
from pydantic import BaseModel, Field
from src.utils import consts
class LinkRequest(BaseModel):
cmd: str = "get"
@@ -26,9 +28,14 @@ class Solution(BaseModel):
response: str
@classmethod
def empty(cls):
def invalid(cls, url: str):
"""
Return an empty Solution with default values.
Useful for returning an error response.
"""
return cls(
url="",
url=url,
status=HTTPStatus.INTERNAL_SERVER_ERROR,
cookies=[],
userAgent="",
@@ -43,19 +50,23 @@ class LinkResponse(BaseModel):
solution: Solution
startTimestamp: int # noqa: N815 # Ignore to preserve compatibility
endTimestamp: int = int(time.time() * 1000) # noqa: N815 # Ignore to preserve compatibility
version: str = "3.3.21" # TODO: Implement versioning
version: str = consts.VERSION
@classmethod
def invalid(cls):
def invalid(cls, url: str):
"""
Return an invalid LinkResponse with default error values.
This method is used to generate a response indicating an invalid request.
"""
return cls(
status="error",
message="Invalid request",
solution=Solution.empty(),
solution=Solution.invalid(url),
startTimestamp=int(time.time() * 1000),
endTimestamp=int(time.time() * 1000),
version="3.3.21",
)
class NoChromeExtensionError(Exception):
"""No chrome extention found."""
"""No chrome extension found."""
+24
View File
@@ -1,9 +1,33 @@
import logging
import os
def get_version_from_env():
"""
Retrieve the version from the environment variable 'VERSION'.
This function checks the 'VERSION' environment variable for a value
that starts with 'v' and returns the version without the prefix.
Returns:
str | None: The version string without the 'v' prefix, or None if
the 'VERSION' environment variable is not set or does not start
with 'v'.
"""
version_env = os.getenv("VERSION")
if not version_env or not version_env.startswith("v"):
return None
return version_env.removeprefix("v")
LOG_LEVEL = os.getenv("LOG_LEVEL") or "INFO"
LOG_LEVEL = logging.getLevelNamesMapping()[LOG_LEVEL.upper()]
VERSION = get_version_from_env() or "unknown"
CHALLENGE_TITLES = [
# Cloudflare
"Just a moment...",
Executable
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
uv run pytest --retries 2 -n auto --xvfb
-7
View File
@@ -12,19 +12,12 @@ client = TestClient(app)
test_websites = [
"https://btmet.com/",
"https://ext.to/",
]
# Not testable on github actions
github_restricted = [
"https://www.ygg.re/",
"https://extratorrent.st/",
"https://idope.se/",
"https://speed.cd/login",
]
# if os.getenv("GITHUB_ACTIONS") != "true":
test_websites.extend(github_restricted)
@pytest.mark.parametrize("website", test_websites)
def test_bypass(website: str):
Generated
+1450
View File
File diff suppressed because it is too large Load Diff