From bc529e591526e1a88793698636d590620e88fba0 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 9 Aug 2026 19:45:44 +0200 Subject: [PATCH 1/9] fix(ci): use slice-free gha cache scopes for cross-job reuse - test job: scope x64 -> amd64 to match build matrix amd64 leg - build job: scope ${{ matrix.platform }} -> ${{ steps.vars.outputs.SURFIX }} (yields amd64/arm64), avoiding the gha backend's / path-separator bug that mangled scope=linux/arm64 and broke arm64 cache reuse test (amd64) and build-amd64 now share scope=amd64 so build reuses the app/base layers the test job cached earlier in the same run. build-arm64 gets a working scope=arm64 that persists across runs. --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6180075..4a68e0b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -65,9 +65,9 @@ jobs: with: context: . platforms: linux/amd64 - cache-from: type=gha,scope=x64 + cache-from: type=gha,scope=amd64 pull: true - cache-to: type=gha,mode=max,scope=x64 + cache-to: type=gha,mode=max,scope=amd64 target: test build: @@ -135,8 +135,8 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ matrix.platform }} - cache-from: type=gha,scope=${{ matrix.platform }} - cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + cache-from: type=gha,scope=${{ steps.vars.outputs.SURFIX }} + cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.SURFIX }} build-args: | GITHUB_BUILD=true VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.sha }} From 1801eaa40ccda72aa70644bc76a053cd001933df Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 9 Aug 2026 19:58:48 +0200 Subject: [PATCH 2/9] fix(ci): scope push trigger to main to avoid duplicate runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit push: branches: ["*"] matched feature branches, so every push to a branch with an open PR fired both a 'push' and a 'pull_request' event. Their concurrency groups differ (refs/heads/ vs refs/pull//merge), so cancel-in-progress could not dedup them — the full multi-arch build ran twice on each push, doubling CI minutes. Scope push to branches: ["main"]; pull_request remains the validator for feature branches. Tag pushes (v*.*.*), schedule, and workflow_dispatch are under separate filters and are unaffected. --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4a68e0b..8ad744f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,7 +9,7 @@ on: schedule: - cron: "25 0 * * *" push: - branches: ["*"] + branches: ["main"] # Publish semver tags as releases. tags: ["v*.*.*"] paths: From bdd59d7e600268f04c5c55ef5413f07219118e9d Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 9 Aug 2026 20:23:12 +0200 Subject: [PATCH 3/9] ci: retrigger cache test (run 2) From 221f27acca295eb56bb6254e132241253c1008f1 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 9 Aug 2026 21:17:13 +0200 Subject: [PATCH 4/9] fix(ci): hoist ARG VERSION to final stage to stop cache busting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of remaining cache misses: the base stage declared ARG VERSION, and the build job passed VERSION=${{ github.sha }}. Since VERSION changes every commit, every base/app layer cache key changed with it — so layers rebuilt every run regardless of scope. Additionally the test job passed no build-args while the build job passed GITHUB_BUILD=true + VERSION, so test's cached base/app layers had different keys from build's — cross-job reuse never hit either. Fix: - Dockerfile: move ARG VERSION / ENV VERSION from base to the final runtime stage (FROM app). VERSION is only read at runtime by src.consts via Pydantic settings; base/app layers don't use it. base/app now cache without per-commit VERSION variation. - workflow: pass --build-arg GITHUB_BUILD=true in the test step so test and build share identical base/app cache keys (cross-job reuse). VERSION is intentionally NOT passed to the test job: the test stage (FROM app AS test) doesn't read VERSION, and omitting it keeps the base/app cache keys identical between test and build. --- .github/workflows/docker-publish.yml | 2 ++ Dockerfile | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8ad744f..7ef231d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -69,6 +69,8 @@ jobs: pull: true cache-to: type=gha,mode=max,scope=amd64 target: test + build-args: | + GITHUB_BUILD=true build: needs: test diff --git a/Dockerfile b/Dockerfile index 9b02061..500af3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,9 @@ # cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch). FROM ubuntu:24.04 AS base -ARG GITHUB_BUILD=false \ - VERSION +ARG GITHUB_BUILD=false ENV GITHUB_BUILD=${GITHUB_BUILD}\ - VERSION=${VERSION}\ - DEBIAN_FRONTEND=noninteractive \ PYTHONUNBUFFERED=1 \ # prevents python creating .pyc files PYTHONDONTWRITEBYTECODE=1 \ @@ -57,6 +54,11 @@ RUN \ uv run pytest --retries 3 FROM app +# VERSION is a runtime-only env var (read by src.consts via Pydantic settings). +# Declared here, not in base, so base/app layer cache keys don't depend on the +# per-commit SHA — that would bust the cache every run. +ARG VERSION +ENV VERSION=${VERSION} USER 1000 EXPOSE $PORT HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health" From 7e1a5d4329e874cdfda99a9012a32165aa09f095 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 9 Aug 2026 21:35:14 +0200 Subject: [PATCH 5/9] =?UTF-8?q?ci:=20retrigger=20cache=20test=20(run=202?= =?UTF-8?q?=20=E2=80=94=20verify=20arm64=20self-reuse)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From baad431605228642f4dea5f150e485fb963f54c7 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Mon, 10 Aug 2026 01:22:09 +0200 Subject: [PATCH 6/9] chore(ci): drop VERSION cache-comment from final stage --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 500af3a..1124ed9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,9 +54,6 @@ RUN \ uv run pytest --retries 3 FROM app -# VERSION is a runtime-only env var (read by src.consts via Pydantic settings). -# Declared here, not in base, so base/app layer cache keys don't depend on the -# per-commit SHA — that would bust the cache every run. ARG VERSION ENV VERSION=${VERSION} USER 1000 From 8ef4c6224997fc8ddab266eef67d3b19a839f3f8 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 9 Aug 2026 23:39:41 +0200 Subject: [PATCH 7/9] fix: detect Cloudflare challenges regardless of language (#385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare localizes its interstitial page title per visitor language (e.g. Polish "Cierpliwości..." served by 1337x.to), so the hard-coded ["Just a moment..."] title check missed every non-English visitor: Byparr returned the raw challenge page (HTTP 403, no cf_clearance cookie, no "Challenge detected" log) and Prowlarr reported "Unable to access 1337x.to, blocked by CloudFlare Protection." (issue #385, still open on 3.0.1 after the compression fix). Replace the title-based gate with the playwright-captcha library's own language-independent DOM detection (detect_cloudflare_challenge), which matches Cloudflare's challenge scripts directly: - interstitial: script[src*="/cdn-cgi/challenge-platform/"] - turnstile: input[name="cf-turnstile-response"], script[src*="challenges.cloudflare.com/turnstile/v0"] Both selectors match the live 1337x "Cierpliwości..." interstitial. The navigation/detect/solve flow lives in _navigate_and_solve(); the timeout-to-408 translation is inlined at the call site in read_item. The now-unused title map is removed from src/consts.py. Verified live (built image): "Challenge detected" now fires on 1337x (0 -> 1 in logs) where the title check never fired; example.com negative control returns 200 with no challenge path entered. End-to-end clearing still depends on the requester's public IP (README caveat). --- src/consts.py | 11 ------ src/endpoints.py | 99 +++++++++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 46 deletions(-) diff --git a/src/consts.py b/src/consts.py index 207139b..f57b592 100644 --- a/src/consts.py +++ b/src/consts.py @@ -3,8 +3,6 @@ import sys from pydantic_settings import BaseSettings, SettingsConfigDict -from playwright_captcha import CaptchaType - class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", extra="ignore") @@ -44,12 +42,3 @@ BLOCK_MEDIA = settings.block_media RETURN_ONLY_COOKIES = settings.return_only_cookies OWUI_API_KEY = settings.owui_api_key - -CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = { - # Cloudflare - CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."], -} - -CHALLENGE_TITLES = [ - title for titles in CHALLENGE_TITLES_MAP.values() for title in titles -] diff --git a/src/endpoints.py b/src/endpoints.py index 714cb4f..346b1b4 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -9,8 +9,10 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse from playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright_captcha import CaptchaType +from playwright_captcha.solvers.click.cloudflare.utils.detection import ( + detect_cloudflare_challenge, +) -from src.consts import CHALLENGE_TITLES from src.models import ( HealthcheckResponse, LinkRequest, @@ -109,37 +111,9 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await dep.page.route("**/*", strip_csp_route) try: - page_request = await dep.page.goto( - request.url, timeout=timer.remaining() * 1000 + challenge_detected, page_html, page_request, status = ( + await _navigate_and_solve(dep, request, timer) ) - status = page_request.status if page_request else HTTPStatus.OK - await dep.page.wait_for_load_state( - state="domcontentloaded", timeout=timer.remaining() * 1000 - ) - - if await dep.page.title() in CHALLENGE_TITLES: - logger.info("Challenge detected, attempting to solve...") - # Solve the captcha - await wait_for( - dep.solver.solve_captcha( # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType] - captcha_container=dep.page, - captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL, - wait_checkbox_attempts=1, - wait_checkbox_delay=0.5, - ), - timeout=timer.remaining(), - ) - status = HTTPStatus.OK - logger.debug("Challenge solved successfully.") - else: - try: - await dep.page.wait_for_load_state( - "networkidle", timeout=timer.remaining() * 1000 - ) - except PlaywrightTimeoutError: - logger.info( - "networkidle timed out after domcontentloaded; continuing with loaded page" - ) except (TimeoutError, PlaywrightTimeoutError) as e: logger.error("Timed out while loading the page or solving the challenge") raise HTTPException( @@ -154,8 +128,11 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: if request.return_only_cookies: response_content = "" - elif page_request and page_request.headers.get("content-type", "").startswith( - "application/pdf" + elif ( + page_request + and page_request.headers.get("content-type", "").startswith( + "application/pdf" + ) ): content_type = "application/pdf" try: @@ -164,11 +141,17 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: await fetch_response.body() ).decode("ascii") except Exception: - logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML") + logger.exception( + "Failed to fetch PDF bytes, falling back to viewer HTML" + ) content_type = "text/html" response_content = await dep.page.content() else: - response_content = await dep.page.content() + response_content = ( + page_html + if page_html is not None and not challenge_detected + else await dep.page.content() + ) return LinkResponse( message="Success", @@ -183,3 +166,49 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse: ), start_timestamp=start_time, ) + + +async def _navigate_and_solve( + dep: BrowserDep, + request: LinkRequest, + timer: TimeoutTimer, +) -> tuple[bool, str | None, object, HTTPStatus]: + page_html: str | None = None + page_request = await dep.page.goto( + request.url, timeout=timer.remaining() * 1000 + ) + status = page_request.status if page_request else HTTPStatus.OK + await dep.page.wait_for_load_state( + state="domcontentloaded", timeout=timer.remaining() * 1000 + ) + + challenge_active = ( + await detect_cloudflare_challenge(dep.page, "interstitial") + or await detect_cloudflare_challenge(dep.page, "turnstile") + ) + if not challenge_active: + page_html = await dep.page.content() + try: + await dep.page.wait_for_load_state( + "networkidle", timeout=timer.remaining() * 1000 + ) + except PlaywrightTimeoutError: + logger.info( + "networkidle timed out after domcontentloaded; " + "continuing with loaded page" + ) + return False, page_html, page_request, status + + logger.info("Challenge detected, attempting to solve...") + await wait_for( + dep.solver.solve_captcha( # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType] + captcha_container=dep.page, + captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL, + wait_checkbox_attempts=1, + wait_checkbox_delay=0.5, + ), + timeout=timer.remaining(), + ) + status = HTTPStatus.OK + logger.debug("Challenge solved successfully.") + return True, page_html, page_request, status From 8cb5770b841af6d6c8c2bf70f137876d7472f691 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Mon, 10 Aug 2026 22:53:42 +0200 Subject: [PATCH 8/9] feat: add BROWSER_LOCALE env to override browser language --- README.md | 7 +++++++ src/consts.py | 3 +++ src/utils.py | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 74898c4..16e4884 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,13 @@ | `PROXY_USERNAME` | None | Username for proxy authentication. | | `PROXY_PASSWORD` | None | Password for proxy authentication. | | `OWUI_API_KEY` | None | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI. | +| `BROWSER_LOCALE` | None | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. | + +#### Browser language + +Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag like `en-US`, `de-DE`, `fr-FR`, `pl-PL`, or `zh-CN` to fix the browser's language and `Accept-Language` header. When unset, Byparr derives the locale from the egress country (e.g. a French proxy → `fr-FR`), keeping the browser language consistent with the exit IP. + +Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`. ## Proxy Recommendation diff --git a/src/consts.py b/src/consts.py index 207139b..a1f123f 100644 --- a/src/consts.py +++ b/src/consts.py @@ -24,6 +24,7 @@ class Settings(BaseSettings): block_media: bool = False return_only_cookies: bool = False owui_api_key: str | None = None + browser_locale: str | None = None settings = Settings() @@ -45,6 +46,8 @@ RETURN_ONLY_COOKIES = settings.return_only_cookies OWUI_API_KEY = settings.owui_api_key +BROWSER_LOCALE = settings.browser_locale + CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = { # Cloudflare CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."], diff --git a/src/utils.py b/src/utils.py index 5a23d86..e34bca9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -13,6 +13,7 @@ from playwright_captcha import ( from pydantic import BaseModel, Field from src.consts import ( + BROWSER_LOCALE, LOG_LEVEL, MAX_ATTEMPTS, PROXY_PASSWORD, @@ -94,7 +95,7 @@ async def get_browser( headless=True, proxy=proxy_config, humanize=True, - locale="auto", + locale=BROWSER_LOCALE or "auto", ) as browser_raw: # InvisiblePlaywright yields a Browser instance browser = cast("Browser", browser_raw) From c38a6f4e8586e68f061ea00a2091c7381d835d68 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Mon, 10 Aug 2026 23:26:14 +0200 Subject: [PATCH 9/9] fix(docker): keep uv Python out of tmpfs-mounted /tmp HOME=/tmp put the uv-managed Python at /tmp/.local/share/uv, so a tmpfs mount on /tmp (e.g. compose tmpfs: /tmp) wiped the interpreter at container start, leaving the /app/.venv/bin/python symlink dangling and startup failing with 'exec /app/.venv/bin/python failed: No such file or directory' (#389). Move HOME to /home/byparr and apply the OpenShift permission pattern (owner uid 1000, group 0, group=user) so both the default user and arbitrary-UID runtimes (docker run --user, OpenShift) can write to it. Apply the same pattern to /cache, where invisible_playwright keeps runtime browser/profile data and which arbitrary UIDs previously could not write. Fixes #389 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1124ed9..920c52e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV GITHUB_BUILD=${GITHUB_BUILD}\ UV_LINK_MODE=copy \ PORT=8191 \ XDG_CACHE_HOME=/cache \ - HOME=/tmp + HOME=/home/byparr RUN apt-get update &&\ apt-get install -y --no-install-recommends curl ca-certificates git tini &&\ @@ -44,9 +44,9 @@ RUN mkdir -p /cache &&\ COPY . . -# Make app and cache world-readable; cache must be writable for runtime browser/profile data -RUN chmod -R o+rX /app /cache &&\ - chmod -R o+w /cache +RUN mkdir -p /home/byparr &&\ + chmod -R o+rX /app &&\ + chmod -R a+rwX /cache /home/byparr FROM app AS test RUN \