mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:05:20 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 CaliBrain
|
9b1d4322b7 | ||
 dependabot[bot]
|
d7a21ea248 | ||
|
dependabot[bot]
|
c487c6c483 | ||
|
CaliBrain
|
404e8cc5c5 | ||
 Alex SchittkoandCaliBrain
|
40d6a179b7 | ||
|
CaliBrain
|
02a45c1a12 | ||
|
dependabot[bot]
|
4c78a2f3a0 | ||
|
Alex Schittko
|
8b4230ee9f | ||
|
dependabot[bot]
|
b57925c39d | ||
|
dependabot[bot]
|
b6efed95ab | ||
|
dependabot[bot]
|
44a261e857 | ||
|
dependabot[bot]
|
14bd50bfd1 | ||
|
dependabot[bot]andCaliBrain
|
4f123977ff | ||
|
dependabot[bot]
|
e3b05d6030 | ||
|
dependabot[bot]
|
995d8d6631 | ||
|
dependabot[bot]
|
9a27334d9c | ||
|
dependabot[bot]
|
76c55a488c | ||
|
dependabot[bot]
|
08d0a04cf2 | ||
|
CaliBrain
|
36f9a52987 | ||
|
CaliBrain
|
1536b91a46 | ||
|
CaliBrain
|
fea63269e3 | ||
|
CaliBrain
|
7886ba7038 | ||
|
CaliBrain
|
1c19326bd0 | ||
|
CaliBrain
|
2f70ed36e4 | ||
|
dependabot[bot]
|
8d54cff383 | ||
|
dependabot[bot]andCaliBrain
|
63fa6213da | ||
|
CaliBrain
|
61b3eaf351 | ||
|
dependabot[bot]
|
7158b79146 | ||
|
dependabot[bot]
|
3c1b79989a | ||
|
dependabot[bot]
|
e41c1d4e8e | ||
  Hrachya ShaginyanandZo Bot
|
ec7e3482bb | ||
|
dependabot[bot]
|
05cd384efc | ||
|
dependabot[bot]
|
880cc4a299 |
@@ -67,7 +67,7 @@ jobs:
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
@@ -79,6 +79,12 @@ jobs:
|
||||
- name: Extract metadata for ${{ matrix.target }} image
|
||||
id: meta
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
env:
|
||||
# Annotate both the per-platform manifests and the multi-arch image
|
||||
# index. The index level is what manifest-list consumers (Renovate's
|
||||
# minimumReleaseAge soak check, provenance/SBOM tooling) read for the
|
||||
# standard org.opencontainers.image.* annotations, including `created`.
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
|
||||
tags: |
|
||||
@@ -105,10 +111,11 @@ jobs:
|
||||
RELEASE_VERSION=${{ github.ref_name }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
annotations: ${{ steps.meta.outputs.annotations }}
|
||||
|
||||
- name: Generate artifact attestation for ${{ matrix.target }} image
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
|
||||
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
language: [python, javascript-typescript]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
name: E2E Platform
|
||||
|
||||
# Hermetic end-to-end matrix: boots the app under test against mock
|
||||
# Anna's Archive / Cloudflare / bypasser / DNS / proxy / Tor / real torrent
|
||||
# clients and runs the cluster suite under each config profile.
|
||||
#
|
||||
# On a PR that touches relevant code, this runs a fast core subset *and* the heavy
|
||||
# `full` profile (real Chrome solving Cloudflare + DoH + real qBittorrent). The
|
||||
# `e2e-required` job aggregates them into ONE status check — make that check a
|
||||
# required status check in branch protection to block merges on any e2e failure
|
||||
# (see tests/e2e/platform/README.md "Gating PRs").
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: "0 4 * * *" # nightly full matrix
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-platform-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Detect whether anything that affects the e2e platform changed. This lets the
|
||||
# required check always report (never stuck "pending") while only spending CI on
|
||||
# PRs that can actually break the e2e stack.
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
relevant: ${{ steps.filter.outputs.relevant }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: dorny/paths-filter@v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
relevant:
|
||||
- 'shelfmark/**'
|
||||
- 'entrypoint.sh'
|
||||
- 'tor.sh'
|
||||
- 'Dockerfile'
|
||||
- 'tests/e2e/platform/**'
|
||||
- '.github/workflows/e2e-platform.yml'
|
||||
|
||||
select-profiles:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
profiles: ${{ steps.pick.outputs.profiles }}
|
||||
steps:
|
||||
- id: pick
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo 'profiles=["baseline","bypasser-external","dns-blocked"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'profiles=["baseline","bypasser-external","bypasser-disabled","dns-manual","dns-blocked","dns-doh","proxy-http","proxy-socks","tor","client-transmission","client-deluge","client-qbittorrent-delayed"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
e2e:
|
||||
needs: select-profiles
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }}
|
||||
name: e2e (${{ matrix.profile }})
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: make install-python-dev
|
||||
- name: Run e2e platform (${{ matrix.profile }})
|
||||
run: tests/e2e/platform/run-e2e.sh env/${{ matrix.profile }}.env
|
||||
- name: Dump shelfmark logs on failure
|
||||
if: failure()
|
||||
run: cat tests/e2e/platform/.state/shelfmark.${{ matrix.profile }}.log || true
|
||||
|
||||
# Heavy "everything real" job: real Chrome internal bypasser solving Cloudflare +
|
||||
# DoH + real qBittorrent webseed download. Runs on relevant PRs and nightly.
|
||||
e2e-full:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
name: e2e (full — real Chrome + qBittorrent)
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: make install-python-dev
|
||||
- name: Run full pipeline
|
||||
run: tests/e2e/platform/run-e2e.sh env/full.env
|
||||
- name: Dump logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
cat tests/e2e/platform/.state/shelfmark.full.log || true
|
||||
docker logs e2e-qbittorrent || true
|
||||
|
||||
# Single aggregated gate. ALWAYS runs (so a required check never hangs "pending"
|
||||
# on unrelated PRs) and FAILS if any e2e job failed/was cancelled. Make THIS the
|
||||
# required status check in branch protection.
|
||||
e2e-required:
|
||||
name: e2e required
|
||||
needs: [e2e, e2e-full]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Aggregate e2e results
|
||||
run: |
|
||||
matrix='${{ needs.e2e.result }}'
|
||||
full='${{ needs.e2e-full.result }}'
|
||||
echo "e2e matrix=$matrix, e2e-full=$full"
|
||||
# success or skipped (unrelated PR) is OK; failure/cancelled blocks.
|
||||
for r in "$matrix" "$full"; do
|
||||
if [ "$r" = "failure" ] || [ "$r" = "cancelled" ]; then
|
||||
echo "::error::An e2e platform job did not pass — blocking."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "All e2e platform jobs passed (or were skipped as not relevant)."
|
||||
@@ -166,6 +166,10 @@ ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# ...but the e2e platform test profiles live in an env/ dir and must be tracked
|
||||
!tests/e2e/platform/env/
|
||||
!tests/e2e/platform/env/*.env
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
+49
-11
@@ -4,7 +4,7 @@ ARG BUILDPLATFORM
|
||||
ARG BUILDARCH
|
||||
|
||||
# Frontend build stage.
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:fb71d01345f11b708a3553c66e7c74074f2d506400ea81973343d915cb64eef0 AS frontend-builder
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS frontend-builder
|
||||
|
||||
# Helpful debug output to see what platforms BuildKit thinks it's using
|
||||
RUN echo "BUILDPLATFORM=$BUILDPLATFORM BUILDARCH=$BUILDARCH TARGETPLATFORM=$TARGETPLATFORM TARGETARCH=$TARGETARCH"
|
||||
@@ -25,7 +25,7 @@ COPY src/frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.5-slim@sha256:c845af9399020c7e562969a13689e929074a10fd057acd1b1fad06a2fb068e97 AS base
|
||||
FROM python:3.14.6-slim@sha256:b877e50bd90de10af8d82c57a022fc2e0dc731c5320d762a27986facfc3355c1 AS base
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 /uv /uvx /bin/
|
||||
|
||||
@@ -72,7 +72,12 @@ RUN apt-get update && \
|
||||
# --- Tor support (activated via USING_TOR=true) ---
|
||||
tor \
|
||||
supervisor \
|
||||
iptables && \
|
||||
iptables \
|
||||
# --- WireGuard support (activated via USING_WIREGUARD=true) ---
|
||||
wireguard-tools \
|
||||
iproute2 \
|
||||
procps \
|
||||
ca-certificates && \
|
||||
# Configure iptables alternatives for tor.sh compatibility
|
||||
update-alternatives --set iptables /usr/sbin/iptables-legacy && \
|
||||
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
|
||||
@@ -131,7 +136,7 @@ RUN mkdir -p \
|
||||
ln -s /tmp/shelfmark/seleniumbase/archived_files /app/archived_files && \
|
||||
chown -R 1000:1000 /config /books /home/shelfmark /tmp/shelfmark /var/log/shelfmark && \
|
||||
chmod -R a+rX /app && \
|
||||
chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
|
||||
chmod +x /app/entrypoint.sh /app/tor.sh /app/wireguard.sh /app/genDebug.sh
|
||||
|
||||
# Expose the application port
|
||||
EXPOSE ${FLASK_PORT}
|
||||
@@ -147,21 +152,39 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
||||
|
||||
FROM base AS shelfmark
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
# --- Chromium (PINNED to 149.0.7827.196) ---
|
||||
# Debian's chromium 150.0.7871.46-1~deb13u1 security update (trixie-security,
|
||||
# 2026-07-05) no longer opens the DevTools remote-debugging TCP port at all
|
||||
# (no listener, no DevToolsActivePort file, even with a custom --user-data-dir;
|
||||
# the RemoteDebuggingAllowed policy does not restore it). The SeleniumBase
|
||||
# Pure-CDP driver connects through that port (/json/version), so with 150 every
|
||||
# internal bypass dies with "Pure CDP browser startup failed" and all
|
||||
# CF-gated downloads fail. Install the last working version from
|
||||
# snapshot.debian.org until the bypasser can talk to Chromium >= 150 (e.g.
|
||||
# pipe-based DevTools / UC mode) or seleniumbase ships a fix.
|
||||
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
|
||||
# This flag is set in internal_bypasser.py _get_browser_args()
|
||||
ARG CHROMIUM_VERSION=149.0.7827.196-1~deb13u1
|
||||
ARG CHROMIUM_SNAPSHOT=20260704T000000Z
|
||||
|
||||
RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/${CHROMIUM_SNAPSHOT}/ trixie-security main" \
|
||||
> /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
|
||||
apt-get update -o Acquire::Retries=5 && \
|
||||
apt-get install -y --no-install-recommends -o Acquire::Retries=5 \
|
||||
# For dumb display
|
||||
xvfb \
|
||||
# For screen recording
|
||||
ffmpeg \
|
||||
# --- Chromium (unpinned - uses latest from Debian repos) ---
|
||||
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
|
||||
# This flag is set in internal_bypasser.py _get_browser_args()
|
||||
chromium \
|
||||
chromium-common \
|
||||
chromium=${CHROMIUM_VERSION} \
|
||||
chromium-common=${CHROMIUM_VERSION} \
|
||||
# For tkinter (pyautogui)
|
||||
python3-tk \
|
||||
# For RAR extraction
|
||||
unrar-free && \
|
||||
# Keep apt from "upgrading" chromium past the pin inside derived images
|
||||
printf 'Package: chromium chromium-common\nPin: version %s\nPin-Priority: 1001\n' "${CHROMIUM_VERSION}" \
|
||||
> /etc/apt/preferences.d/chromium-pin && \
|
||||
rm /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
|
||||
# Create symlink so rarfile library can find unrar
|
||||
ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
|
||||
# Cleanup APT cache
|
||||
@@ -173,6 +196,21 @@ RUN apt-get update && \
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --locked --no-default-groups --extra browser
|
||||
|
||||
# Deterministically resolve the Xlib namespace collision.
|
||||
# pyautogui/mouseinfo pull the stale `python3-xlib` (0.15, 2014), while the
|
||||
# `--extra browser` set pulls `python-xlib` (0.33). Both packages install into
|
||||
# the same top-level `Xlib/` namespace, so whichever lands last wins. When the
|
||||
# 2014 build wins, `Xlib.X` is missing `FamilyServerInterpreted`, which the
|
||||
# SeleniumBase Pure-CDP driver requires at browser startup -> every bypass fails
|
||||
# with "module 'Xlib.X' has no attribute 'FamilyServerInterpreted'" and no
|
||||
# Cloudflare/DDoS-Guard protected download can complete. Drop the stale package
|
||||
# and force python-xlib 0.33 to own the namespace. pyautogui runs fine against
|
||||
# 0.33 (superset API).
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip uninstall --python /app/.venv/bin/python python3-xlib && \
|
||||
uv pip install --python /app/.venv/bin/python --reinstall python-xlib==0.33 && \
|
||||
/app/.venv/bin/python -c "import Xlib.X; assert hasattr(Xlib.X, 'FamilyServerInterpreted'), 'Xlib.X.FamilyServerInterpreted missing after fix'; print('Xlib namespace OK:', Xlib.__version__)"
|
||||
|
||||
# uv is only needed while building the image.
|
||||
RUN rm -f /usr/bin/uv /usr/bin/uvx
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov checks fix
|
||||
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov e2e-platform e2e-platform-profile e2e-platform-matrix e2e-platform-full e2e-platform-build checks fix
|
||||
|
||||
# Frontend directory
|
||||
FRONTEND_DIR := src/frontend
|
||||
@@ -38,6 +38,10 @@ help:
|
||||
@echo " python-checks - Run all Python static analysis checks"
|
||||
@echo " python-test - Run unit tests"
|
||||
@echo " python-test-cov - Run unit tests with coverage report"
|
||||
@echo " e2e-platform - Run e2e docker platform (baseline profile)"
|
||||
@echo " e2e-platform-profile PROFILE=<name> - Run e2e platform for one profile"
|
||||
@echo " e2e-platform-matrix - Run e2e platform across all config profiles"
|
||||
@echo " e2e-platform-full - Run heavy 'full' profile (real Chrome bypasser + DoH + real qBittorrent)"
|
||||
@echo " clean - Remove node_modules and build artifacts"
|
||||
@echo ""
|
||||
@echo "Backend (Docker):"
|
||||
@@ -127,6 +131,30 @@ python-test-cov:
|
||||
@echo "Running tests with coverage..."
|
||||
uv run pytest tests/ -x --tb=short -m "not integration and not e2e" --cov --cov-report=term-missing
|
||||
|
||||
# E2E docker platform: hermetic stack (mock AA/Cloudflare/bypasser/DNS/proxy/Tor)
|
||||
# exercised across config profiles. See tests/e2e/platform/README.md.
|
||||
E2E_PLATFORM_DIR := tests/e2e/platform
|
||||
|
||||
e2e-platform:
|
||||
@echo "Running e2e platform (baseline profile)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/baseline.env
|
||||
|
||||
e2e-platform-profile:
|
||||
@echo "Running e2e platform (profile=$(PROFILE))..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/$(PROFILE).env
|
||||
|
||||
e2e-platform-matrix:
|
||||
@echo "Running e2e platform matrix (all profiles)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-matrix.sh
|
||||
|
||||
e2e-platform-build:
|
||||
@echo "Pre-building e2e platform images once (reused across profiles)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./build-images.sh
|
||||
|
||||
e2e-platform-full:
|
||||
@echo "Running e2e platform FULL profile (real Chrome bypasser + DoH + real qBittorrent)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/full.env
|
||||
|
||||
# Frontend linting
|
||||
frontend-lint:
|
||||
@echo "Running Oxlint..."
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# Routes all traffic through a WireGuard tunnel - requires root startup.
|
||||
#
|
||||
# Mount your wg-quick config at /config/wg0.conf (read-only is fine). All
|
||||
# non-LAN egress is forced through the tunnel by an iptables kill-switch, so if
|
||||
# the tunnel drops, external traffic fails closed. LAN ranges (WebUI + internal
|
||||
# download clients like Prowlarr / qBittorrent) stay reachable off-tunnel.
|
||||
services:
|
||||
shelfmark-wireguard:
|
||||
image: ghcr.io/calibrain/shelfmark:latest
|
||||
environment:
|
||||
FLASK_PORT: 8084
|
||||
# Quoted so it is passed as the literal string "true": entrypoint.sh compares
|
||||
# $USING_WIREGUARD against "true", and some Compose implementations stringify
|
||||
# a bare YAML boolean as "True", which would silently NOT enable WireGuard.
|
||||
USING_WIREGUARD: "true"
|
||||
# Path to the mounted wg-quick config (default shown).
|
||||
WIREGUARD_CONFIG: /config/wg0.conf
|
||||
# CIDRs kept OFF the tunnel so the WebUI and internal clients stay reachable.
|
||||
LAN_NETWORK: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
# WireGuard needs the module/kernel routing; NET_ADMIN covers wg-quick.
|
||||
sysctls:
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
# Disable IPv6 in the container so the kill-switch can guarantee no IPv6
|
||||
# leak path on kernels/containers without a usable ip6tables. wireguard.sh
|
||||
# fails closed if IPv6 is neither kill-switched nor disabled. If your host
|
||||
# DOES have a working ip6tables you may omit these (an ip6tables kill-switch
|
||||
# is installed instead); or set WIREGUARD_ALLOW_IPV6_LEAK=true only if the
|
||||
# container genuinely has no IPv6 connectivity.
|
||||
- net.ipv6.conf.all.disable_ipv6=1
|
||||
- net.ipv6.conf.default.disable_ipv6=1
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Default destination for book downloads
|
||||
- /path/to/config:/config # App configuration (put wg0.conf here)
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -0,0 +1,25 @@
|
||||
# Local development - WireGuard variant
|
||||
services:
|
||||
shelfmark-wireguard-dev:
|
||||
extends:
|
||||
file: ./compose/docker-compose.wireguard.yml
|
||||
service: shelfmark-wireguard
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
target: shelfmark
|
||||
environment:
|
||||
# Quoted so they are passed as the literal string "true" (entrypoint.sh and
|
||||
# the app compare against "true"); a bare YAML boolean can be stringified as
|
||||
# "True" by some Compose variants, silently disabling the feature.
|
||||
DEBUG: "true"
|
||||
USING_WIREGUARD: "true"
|
||||
WIREGUARD_CONFIG: /config/wg0.conf
|
||||
volumes:
|
||||
- ./.local/config:/config
|
||||
- ./.local/books:/books
|
||||
- ./.local/log:/var/log/shelfmark
|
||||
- ./.local/tmp:/tmp/shelfmark
|
||||
# Place your wg-quick config at ./.local/config/wg0.conf
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -91,6 +91,10 @@ Example:
|
||||
- Shelfmark can see the same files at `/downloads/books/...`
|
||||
- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
|
||||
|
||||
If the files are copied or synced into Shelfmark on a delay, increase **Completed Path Wait (seconds)**
|
||||
in Settings -> Advanced. The default is 60 seconds; seedbox or remote-sync setups may need a value
|
||||
longer than the sync interval.
|
||||
|
||||
## File Processing Options
|
||||
|
||||
### Transfer Method (Torrent / Usenet Only)
|
||||
|
||||
@@ -7,6 +7,7 @@ This document lists all configuration options that can be set via environment va
|
||||
## Table of Contents
|
||||
|
||||
- [Bootstrap Configuration](#bootstrap-configuration)
|
||||
- [Egress / VPN Routing](#egress--vpn-routing)
|
||||
- [General](#general)
|
||||
- [Search Mode](#search-mode)
|
||||
- [Downloads](#downloads)
|
||||
@@ -145,6 +146,98 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme
|
||||
|
||||
</details>
|
||||
|
||||
## Egress / VPN Routing
|
||||
|
||||
These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.
|
||||
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `USING_WIREGUARD` | Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR. | boolean | `false` |
|
||||
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick configuration file. | string (path) | `/config/wg0.conf` |
|
||||
| `WIREGUARD_INTERFACE` | WireGuard interface name brought up by wg-quick. | string | `wg0` |
|
||||
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable. | string (comma-separated) | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
|
||||
| `WIREGUARD_ENFORCE_DNS` | Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable. | boolean | `true` |
|
||||
| `WIREGUARD_DNS` | Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead. | string (comma-separated) | `unset (uses config DNS = line)` |
|
||||
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface. | boolean | `true` |
|
||||
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel. | boolean | `false` |
|
||||
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK). | boolean | `false` |
|
||||
| `WIREGUARD_STALE_AFTER` | Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel. | number | `180` |
|
||||
|
||||
<details>
|
||||
<summary>Detailed descriptions</summary>
|
||||
|
||||
#### `USING_WIREGUARD`
|
||||
|
||||
Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `WIREGUARD_CONFIG`
|
||||
|
||||
Path to the mounted wg-quick configuration file.
|
||||
|
||||
- **Type:** string (path)
|
||||
- **Default:** `/config/wg0.conf`
|
||||
|
||||
#### `WIREGUARD_INTERFACE`
|
||||
|
||||
WireGuard interface name brought up by wg-quick.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** `wg0`
|
||||
|
||||
#### `LAN_NETWORK`
|
||||
|
||||
Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`
|
||||
|
||||
#### `WIREGUARD_ENFORCE_DNS`
|
||||
|
||||
Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `true`
|
||||
|
||||
#### `WIREGUARD_DNS`
|
||||
|
||||
Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `unset (uses config DNS = line)`
|
||||
|
||||
#### `WIREGUARD_DISABLE_IPV6`
|
||||
|
||||
Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `true`
|
||||
|
||||
#### `WIREGUARD_ALLOW_IPV6_LEAK`
|
||||
|
||||
Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL`
|
||||
|
||||
When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `WIREGUARD_STALE_AFTER`
|
||||
|
||||
Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `180`
|
||||
|
||||
</details>
|
||||
|
||||
## General
|
||||
|
||||
| Variable | Description | Type | Default |
|
||||
@@ -958,6 +1051,7 @@ Comma-separated hosts to bypass proxy (e.g., localhost,127.0.0.1,10.*,*.local)
|
||||
| `CUSTOM_SCRIPT` | Path to a script to run after each successful download. Must be executable. | string | _none_ |
|
||||
| `CUSTOM_SCRIPT_PATH_MODE` | Pass the path to the custom script as an absolute path or relative to the destination folder. | string (choice) | `absolute` |
|
||||
| `CUSTOM_SCRIPT_JSON_PAYLOAD` | Send a JSON payload to the script via stdin. Useful for multi-file imports (audiobooks) or richer metadata without relying on path parsing. | boolean | `false` |
|
||||
| `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT` | How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows. | number | `60` |
|
||||
| `COVERS_CACHE_ENABLED` | Cache book covers on the server for faster loading. | boolean | `true` |
|
||||
| `COVERS_CACHE_TTL` | How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork). | number | `0` |
|
||||
| `COVERS_CACHE_MAX_SIZE_MB` | Maximum disk space for cached covers. Oldest images are removed when limit is reached. | number | `500` |
|
||||
@@ -1038,6 +1132,16 @@ Send a JSON payload to the script via stdin. Useful for multi-file imports (audi
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT`
|
||||
|
||||
**Completed Path Wait (seconds)**
|
||||
|
||||
How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `60`
|
||||
- **Constraints:** min: 0, max: 3600
|
||||
|
||||
#### `COVERS_CACHE_ENABLED`
|
||||
|
||||
**Enable Cover Cache**
|
||||
|
||||
+27
-1
@@ -81,6 +81,13 @@ if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Egress modes are mutually exclusive. Check this BEFORE starting either one so
|
||||
# we never run tor.sh and then abort, leaving a half-configured network stack.
|
||||
if [ "$USING_TOR" = "true" ] && [ "$USING_WIREGUARD" = "true" ]; then
|
||||
echo "USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$USING_TOR" = "true" ]; then
|
||||
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
|
||||
echo "USING_TOR=true requires the container to start as root." >&2
|
||||
@@ -90,6 +97,15 @@ if [ "$USING_TOR" = "true" ]; then
|
||||
./tor.sh
|
||||
fi
|
||||
|
||||
if [ "$USING_WIREGUARD" = "true" ]; then
|
||||
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
|
||||
echo "USING_WIREGUARD=true requires the container to start as root." >&2
|
||||
echo "Non-root mode skips the privileged network setup WireGuard depends on." >&2
|
||||
exit 1
|
||||
fi
|
||||
./wireguard.sh
|
||||
fi
|
||||
|
||||
if [ "$FILE_LOGGING_ENABLED" = "true" ]; then
|
||||
start_file_logging "$LOG_FILE"
|
||||
fi
|
||||
@@ -235,7 +251,11 @@ test_write() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! run_as_target_user sh -c 'echo 0123456789_TEST > "$1"' _ "$test_file"; then
|
||||
# This is a probe: a failure here is expected (e.g. a fresh root-owned bind
|
||||
# mount) and is recovered by the caller via change_ownership + re-probe. Hide
|
||||
# the shell's "Permission denied"/"Read-only file system" stderr so a handled
|
||||
# probe miss doesn't masquerade as a real boot failure in the logs.
|
||||
if ! run_as_target_user sh -c 'echo 0123456789_TEST 2>/dev/null > "$1"' _ "$test_file"; then
|
||||
echo "Failed to write test file in $folder as $USERNAME"
|
||||
return 1
|
||||
fi
|
||||
@@ -448,6 +468,12 @@ else
|
||||
if [ $config_ok -ne 0 ]; then
|
||||
fail_unwritable_config_dir "$CONFIG_PATH"
|
||||
fi
|
||||
|
||||
# The ingest/destination library (default /books) is user data and may be a
|
||||
# bind mount owned by another uid; downloads fail with "Destination not
|
||||
# writable" if the runtime user can't write there. Fix the top-level dir only
|
||||
# (root mode) so we don't recursively chown a potentially huge library.
|
||||
make_writable "${INGEST_DIR:-/books}" root
|
||||
fi
|
||||
|
||||
# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
|
||||
|
||||
+4
-4
@@ -22,25 +22,25 @@ dependencies = [
|
||||
"qbittorrent-api",
|
||||
"transmission-rpc",
|
||||
"authlib>=1.7.2,<1.8",
|
||||
"apprise>=1.11.0",
|
||||
"apprise>=1.12.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
browser = [
|
||||
"pyvirtualdisplay",
|
||||
"pyautogui",
|
||||
"seleniumbase==4.49.10",
|
||||
"seleniumbase==4.50.5",
|
||||
"python-xlib",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.39.7",
|
||||
"basedpyright>=1.39.9",
|
||||
"prek",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-xdist>=3.8.0",
|
||||
"ruff==0.15.17",
|
||||
"ruff==0.15.20",
|
||||
"vulture>=2.14",
|
||||
]
|
||||
|
||||
|
||||
@@ -106,6 +106,16 @@ Environment variables work for initial setup and Docker deployments. They serve
|
||||
| `PUID` / `PGID` | Runtime user/group for the default root-startup flow (also supports legacy `UID`/`GID`) | `1000` / `1000` |
|
||||
| `SEARCH_MODE` | `direct` or `universal` | `universal` |
|
||||
| `USING_TOR` | Enable Tor routing (requires root startup) | `false` |
|
||||
| `USING_WIREGUARD` | Enable WireGuard VPN egress with kill-switch (requires root startup) | `false` |
|
||||
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick config | `/config/wg0.conf` |
|
||||
| `WIREGUARD_INTERFACE` | WireGuard interface name | `wg0` |
|
||||
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI / internal clients stay reachable | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
|
||||
| `WIREGUARD_ENFORCE_DNS` | Pin the resolver (via `WIREGUARD_DNS`, else the config's `DNS =`) so DNS can't silently fall back to an off-tunnel path. Designed for a trusted LAN resolver kept reachable via `LAN_NETWORK` (query leaves over the LAN; download still egresses via the tunnel) — it does **not** force queries through the tunnel. Docker's embedded resolver (`127.0.0.11`) is preserved when present so container-name resolution keeps working; pin its upstream via the container's `dns:` list. Fails closed if no resolver is available or `/etc/resolv.conf` is not writable. | `true` |
|
||||
| `WIREGUARD_DNS` | Explicit resolver(s) to pin (comma/space separated). Use when the VPN's pushed DNS filters domains you need; point at a resolver reachable via the tunnel or an allowed LAN resolver. | _(unset; uses config `DNS =`)_ |
|
||||
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 from the tunnel config (many container kernels lack the ip6tables `raw` table wg-quick needs) and remove IPv6 as a leak surface. | `true` |
|
||||
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue even when an IPv6 kill-switch can't be installed AND IPv6 can't be disabled. Only set if the container has no IPv6 connectivity. | `false` |
|
||||
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | Opt-in off-tunnel WebUI reachability. Default (`false`) keeps the kill-switch strictly fail-closed: the only off-tunnel egress is loopback, the tunnel device and the LAN allowlist. Set `true` only if a **non-LAN** client (e.g. a public reverse proxy on another segment) must reach the WebUI; it permits app-server **replies** (`--sport FLASK_PORT`, conntrack REPLY) off-tunnel — server replies only, never client-initiated egress. LAN clients never need it (covered by `LAN_NETWORK`). | `false` |
|
||||
| `WIREGUARD_STALE_AFTER` | Seconds since the last handshake before the healthcheck bounces the tunnel. | `180` |
|
||||
|
||||
See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
|
||||
|
||||
@@ -141,6 +151,24 @@ docker compose -f docker-compose.tor.yml up -d
|
||||
- Timezone is auto-detected from Tor exit node
|
||||
- Custom DNS/proxy settings are ignored when Tor is active
|
||||
|
||||
#### WireGuard VPN Routing
|
||||
Optional WireGuard support to route all external egress through a VPN tunnel with a fail-closed kill-switch:
|
||||
```bash
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.wireguard.yml
|
||||
# place your wg-quick config where the compose mounts /config, as wg0.conf
|
||||
docker compose -f docker-compose.wireguard.yml up -d
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Requires root startup
|
||||
- Requires `NET_ADMIN` and `NET_RAW` capabilities
|
||||
- Mount a standard wg-quick config at `WIREGUARD_CONFIG` (default `/config/wg0.conf`)
|
||||
- All non-LAN egress is forced through the tunnel; if the tunnel drops, external traffic **fails closed** while LAN ranges (WebUI, Prowlarr, qBittorrent) stay reachable
|
||||
- IPv4 and IPv6 both fail closed. On kernels without a usable `ip6tables`, disable IPv6 for the container (`sysctls: net.ipv6.conf.all.disable_ipv6=1`, as in the compose example) or the container refuses to start rather than risk an IPv6 leak
|
||||
- A supervised healthcheck bounces the tunnel if the handshake goes stale, and refreshes the endpoint allow rules so a roaming/rotated peer endpoint can reconnect
|
||||
- Mutually exclusive with `USING_TOR`
|
||||
- **DNS trust:** `WIREGUARD_DNS` must be a resolver you trust on a trusted network segment. When it is a LAN resolver (kept reachable off-tunnel by `LAN_NETWORK`), the query to that resolver leaves as plaintext UDP/53 on the LAN — the resolver is responsible for encrypting upstream. Two resolver paths exist: (1) when Docker's embedded resolver (`127.0.0.11`) is present it is **preserved** so container names (Prowlarr, qBittorrent) resolve — you MUST pin its upstream to a trusted resolver via the container's compose `dns:` list, since `WIREGUARD_DNS` cannot repoint the embedded resolver from inside the container; (2) otherwise `WIREGUARD_DNS`/the config `DNS =` line is written to `/etc/resolv.conf`. Setting `WIREGUARD_ENFORCE_DNS=false` is a **foot-gun**: with no embedded resolver present the container then uses its inherited resolver, which forwards to the Docker daemon's upstream **off-tunnel**, leaking your DNS. Leave enforcement on unless you have pinned the resolver another way.
|
||||
|
||||
### Lite
|
||||
A lighter image without the built-in browser automation. Ideal for:
|
||||
|
||||
|
||||
@@ -238,6 +238,113 @@ def _generate_bootstrap_env_docs() -> list[str]:
|
||||
return lines
|
||||
|
||||
|
||||
def _generate_egress_env_docs() -> list[str]:
|
||||
"""Generate documentation for VPN/Tor egress environment variables.
|
||||
|
||||
These are startup-only variables consumed by entrypoint.sh / wireguard.sh
|
||||
(before and outside the settings registry) to select and configure the
|
||||
transparent-egress kill-switch. `USING_TOR` has a registry-backed entry
|
||||
under Network and is cross-referenced rather than repeated here so the two
|
||||
mutually exclusive egress modes are discoverable side by side without
|
||||
emitting a duplicate `#### USING_TOR` anchor.
|
||||
"""
|
||||
egress_vars = [
|
||||
{
|
||||
"name": "USING_WIREGUARD",
|
||||
"description": "Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_CONFIG",
|
||||
"description": "Path to the mounted wg-quick configuration file.",
|
||||
"type": "string (path)",
|
||||
"default": "/config/wg0.conf",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_INTERFACE",
|
||||
"description": "WireGuard interface name brought up by wg-quick.",
|
||||
"type": "string",
|
||||
"default": "wg0",
|
||||
},
|
||||
{
|
||||
"name": "LAN_NETWORK",
|
||||
"description": "Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.",
|
||||
"type": "string (comma-separated)",
|
||||
"default": "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_ENFORCE_DNS",
|
||||
"description": "Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.",
|
||||
"type": "boolean",
|
||||
"default": "true",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_DNS",
|
||||
"description": "Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.",
|
||||
"type": "string (comma-separated)",
|
||||
"default": "unset (uses config DNS = line)",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_DISABLE_IPV6",
|
||||
"description": "Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.",
|
||||
"type": "boolean",
|
||||
"default": "true",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_ALLOW_IPV6_LEAK",
|
||||
"description": "Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_ALLOW_WEBUI_OFFTUNNEL",
|
||||
"description": "When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_STALE_AFTER",
|
||||
"description": "Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.",
|
||||
"type": "number",
|
||||
"default": "180",
|
||||
},
|
||||
]
|
||||
|
||||
lines = [
|
||||
"## Egress / VPN Routing",
|
||||
"",
|
||||
"These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.",
|
||||
"",
|
||||
"| Variable | Description | Type | Default |",
|
||||
"|----------|-------------|------|---------|",
|
||||
]
|
||||
|
||||
lines.extend(
|
||||
f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |"
|
||||
for var in egress_vars
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
lines.append("<details>")
|
||||
lines.append("<summary>Detailed descriptions</summary>")
|
||||
lines.append("")
|
||||
|
||||
for var in egress_vars:
|
||||
lines.append(f"#### `{var['name']}`")
|
||||
lines.append("")
|
||||
lines.append(var["description"])
|
||||
lines.append("")
|
||||
lines.append(f"- **Type:** {var['type']}")
|
||||
lines.append(f"- **Default:** `{var['default']}`")
|
||||
lines.append("")
|
||||
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def generate_env_docs() -> str:
|
||||
"""Generate markdown documentation for all environment variables."""
|
||||
# Import settings modules to ensure all settings are registered
|
||||
@@ -282,6 +389,7 @@ def generate_env_docs() -> str:
|
||||
# Generate TOC
|
||||
toc_entries = [
|
||||
"- [Bootstrap Configuration](#bootstrap-configuration)",
|
||||
"- [Egress / VPN Routing](#egress--vpn-routing)",
|
||||
]
|
||||
|
||||
# Ungrouped tabs first
|
||||
@@ -307,6 +415,9 @@ def generate_env_docs() -> str:
|
||||
# Add bootstrap environment variables documentation
|
||||
lines.extend(_generate_bootstrap_env_docs())
|
||||
|
||||
# Add egress / VPN routing (startup-only, shell-driven) documentation
|
||||
lines.extend(_generate_egress_env_docs())
|
||||
|
||||
# Generate documentation for ungrouped tabs
|
||||
for tab in grouped_tabs.get(None, []):
|
||||
lines.extend(_generate_tab_docs(tab))
|
||||
|
||||
@@ -939,7 +939,17 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
|
||||
result_path = (
|
||||
Path(tempfile.gettempdir()) / f"shelfmark-bypass-{os.getpid()}-{time.time_ns()}.json"
|
||||
)
|
||||
payload = {"url": url, "retry": retry, "result_path": str(result_path)}
|
||||
# DNS provider state lives only in the parent's memory (no disk persistence), so the
|
||||
# freshly spawned helper would otherwise pre-resolve AA hostnames against the system
|
||||
# resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
|
||||
# active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
|
||||
# resolver rules.
|
||||
payload = {
|
||||
"url": url,
|
||||
"retry": retry,
|
||||
"result_path": str(result_path),
|
||||
"dns_config": network.get_dns_config(),
|
||||
}
|
||||
env_vars = os.environ.copy()
|
||||
env_vars[_BYPASS_CHILD_ENV] = "1"
|
||||
env_vars = _prepare_child_browser_env(env_vars)
|
||||
@@ -1278,6 +1288,30 @@ def get_bypassed_page(
|
||||
return response_html
|
||||
|
||||
|
||||
def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
|
||||
"""Mirror the parent process's active DNS provider in this helper subprocess.
|
||||
|
||||
DNS state is in-memory only, so a fresh helper defaults to system DNS and would
|
||||
pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
|
||||
that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
|
||||
the same DoH/custom resolver the parent already validated.
|
||||
"""
|
||||
provider = str(dns_config.get("provider") or "").strip().lower()
|
||||
# "auto" means the parent has not rotated off system DNS yet, so the helper's own
|
||||
# default initialization already matches it - nothing to override.
|
||||
if not provider or provider == "auto":
|
||||
return
|
||||
manual_servers = dns_config.get("servers") if provider == "manual" else None
|
||||
try:
|
||||
network.set_dns_provider(
|
||||
provider,
|
||||
manual_servers,
|
||||
use_doh=bool(dns_config.get("doh_enabled")),
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
|
||||
|
||||
|
||||
def _run_child_process() -> int:
|
||||
"""CLI entrypoint used by the Docker helper subprocess."""
|
||||
request = json.loads(sys.stdin.read() or "{}")
|
||||
@@ -1287,6 +1321,10 @@ def _run_child_process() -> int:
|
||||
request.get("retry"), _coerce_positive_int(app_config.MAX_RETRY, 10)
|
||||
)
|
||||
|
||||
dns_config = request.get("dns_config")
|
||||
if isinstance(dns_config, dict):
|
||||
_apply_parent_dns_config(dns_config)
|
||||
|
||||
try:
|
||||
html = get(url, retry=retry)
|
||||
payload = {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
from shelfmark.core.config import config
|
||||
@@ -23,7 +24,7 @@ _BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
|
||||
|
||||
|
||||
def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
|
||||
return f"{base_url}|{username}|{hash(password)}"
|
||||
return f"{base_url}|{username}|{hashlib.sha256(password.encode()).hexdigest()}"
|
||||
|
||||
|
||||
def _get_booklore_select_options(
|
||||
|
||||
@@ -36,6 +36,9 @@ from shelfmark.core.settings_registry import (
|
||||
register_settings,
|
||||
)
|
||||
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
|
||||
|
||||
|
||||
def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate advanced settings before persisting."""
|
||||
@@ -43,6 +46,40 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
timeout_key = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
|
||||
if timeout_key in values:
|
||||
raw_timeout = values.get(timeout_key)
|
||||
if isinstance(raw_timeout, bool):
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Completed Path Wait must be a number of seconds",
|
||||
"values": values,
|
||||
}
|
||||
if raw_timeout is None:
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Completed Path Wait must be a number of seconds",
|
||||
"values": values,
|
||||
}
|
||||
try:
|
||||
timeout_seconds = int(raw_timeout)
|
||||
except TypeError, ValueError:
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Completed Path Wait must be a number of seconds",
|
||||
"values": values,
|
||||
}
|
||||
if timeout_seconds < 0 or timeout_seconds > _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX:
|
||||
return {
|
||||
"error": True,
|
||||
"message": (
|
||||
"Completed Path Wait must be between 0 and "
|
||||
f"{_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX} seconds"
|
||||
),
|
||||
"values": values,
|
||||
}
|
||||
values[timeout_key] = timeout_seconds
|
||||
|
||||
mappings = values.get("PROWLARR_REMOTE_PATH_MAPPINGS")
|
||||
if mappings is None:
|
||||
return {"error": False, "values": values}
|
||||
@@ -1780,6 +1817,18 @@ def advanced_settings() -> list[SettingsField]:
|
||||
title="Remote Path Mappings",
|
||||
description="Map download client paths to paths inside Shelfmark. Needed when volume mounts differ between containers.",
|
||||
),
|
||||
NumberField(
|
||||
key="DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT",
|
||||
label="Completed Path Wait (seconds)",
|
||||
description=(
|
||||
"How long to wait after a torrent or usenet client reports completion "
|
||||
"for the completed file path to become visible to Shelfmark. Increase "
|
||||
"this for seedbox or remote-sync workflows."
|
||||
),
|
||||
default=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT,
|
||||
min_value=0,
|
||||
max_value=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX,
|
||||
),
|
||||
TableField(
|
||||
key="PROWLARR_REMOTE_PATH_MAPPINGS",
|
||||
label="Path Mappings",
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import math
|
||||
import shutil
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
@@ -55,6 +56,19 @@ SECONDS_PER_HOUR = 3600
|
||||
# How long to wait for completed files to appear (seconds)
|
||||
COMPLETED_PATH_RETRY_INTERVAL = 5
|
||||
COMPLETED_PATH_MAX_ATTEMPTS = 12 # 12 attempts * 5s = 60s grace period
|
||||
COMPLETED_PATH_TIMEOUT_SETTING = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
|
||||
COMPLETED_PATH_TIMEOUT_MAX_SECONDS = 3600
|
||||
_RETRYABLE_COMPLETED_PATH_ERRNOS = frozenset(
|
||||
code
|
||||
for code in (
|
||||
errno.ENOENT,
|
||||
getattr(errno, "ESTALE", None),
|
||||
getattr(errno, "EAGAIN", None),
|
||||
getattr(errno, "EBUSY", None),
|
||||
getattr(errno, "ETIMEDOUT", None),
|
||||
)
|
||||
if code is not None
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -69,6 +83,39 @@ class DownloadRequest:
|
||||
ratio_limit: float | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CompletedPathResolution:
|
||||
path: Path | None
|
||||
error: str | None
|
||||
retryable: bool
|
||||
|
||||
|
||||
def _coerce_completed_path_timeout_seconds(value: object, default: float) -> float:
|
||||
if isinstance(value, bool) or value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
parsed = float(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
parsed = float(value.strip())
|
||||
except ValueError:
|
||||
return default
|
||||
else:
|
||||
return default
|
||||
|
||||
if not math.isfinite(parsed) or parsed < 0:
|
||||
return default
|
||||
return min(parsed, float(COMPLETED_PATH_TIMEOUT_MAX_SECONDS))
|
||||
|
||||
|
||||
def _is_retryable_completed_path_probe(error: OSError | None) -> bool:
|
||||
return error is not None and error.errno in _RETRYABLE_COMPLETED_PATH_ERRNOS
|
||||
|
||||
|
||||
def _path_needs_mapping(path: str) -> bool:
|
||||
return (len(path) >= WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":") or "\\" in path
|
||||
|
||||
|
||||
def _diagnose_path_issue(path: str) -> str:
|
||||
"""Analyze a path and return diagnostic hints for common issues.
|
||||
|
||||
@@ -167,6 +214,12 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
"""Maximum attempts when waiting for completed files."""
|
||||
return COMPLETED_PATH_MAX_ATTEMPTS
|
||||
|
||||
def _completed_path_timeout_seconds(self) -> float:
|
||||
"""Total time to wait for completed files to appear on disk."""
|
||||
fallback = self._completed_path_retry_interval() * self._completed_path_max_attempts()
|
||||
configured = config.get(COMPLETED_PATH_TIMEOUT_SETTING, fallback)
|
||||
return _coerce_completed_path_timeout_seconds(configured, fallback)
|
||||
|
||||
def _get_category_for_task(self, client: DownloadClient, task: DownloadTask) -> str | None:
|
||||
"""Get audiobook category if configured and applicable, else None for default."""
|
||||
if not is_audiobook(task.content_type):
|
||||
@@ -387,6 +440,21 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
log_details: bool,
|
||||
) -> tuple[Path | None, str | None]:
|
||||
"""Resolve and validate the completed download path once."""
|
||||
result = self._resolve_download_path_once_detailed(
|
||||
client,
|
||||
download_id,
|
||||
log_details=log_details,
|
||||
)
|
||||
return result.path, result.error
|
||||
|
||||
def _resolve_download_path_once_detailed(
|
||||
self,
|
||||
client: DownloadClient,
|
||||
download_id: str,
|
||||
*,
|
||||
log_details: bool,
|
||||
) -> _CompletedPathResolution:
|
||||
"""Resolve and validate a completed path, including retryability."""
|
||||
try:
|
||||
raw_path = client.get_download_path(download_id)
|
||||
except Exception as e:
|
||||
@@ -402,7 +470,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.debug(
|
||||
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
|
||||
)
|
||||
return None, message
|
||||
return _CompletedPathResolution(None, message, retryable=False)
|
||||
|
||||
if not raw_path:
|
||||
message = (
|
||||
@@ -417,7 +485,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.debug(
|
||||
"Download client returned empty path for %s %s", client.name, download_id
|
||||
)
|
||||
return None, message
|
||||
return _CompletedPathResolution(None, message, retryable=False)
|
||||
|
||||
from shelfmark.core.path_mappings import (
|
||||
get_client_host_identifier,
|
||||
@@ -457,7 +525,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.error(failure_log, *failure_args)
|
||||
else:
|
||||
logger.debug(failure_log, *failure_args)
|
||||
return None, message
|
||||
return _CompletedPathResolution(None, message, retryable=False)
|
||||
|
||||
remapped_exists, remapped_error = _probe_completed_path(remapped)
|
||||
|
||||
@@ -480,7 +548,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
source_path_obj,
|
||||
remapped,
|
||||
)
|
||||
return remapped, None
|
||||
return _CompletedPathResolution(remapped, None, retryable=False)
|
||||
|
||||
message = (
|
||||
f"Remapped path '{remapped}' does not exist. "
|
||||
@@ -499,7 +567,11 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.error(failure_log, *failure_args)
|
||||
else:
|
||||
logger.debug(failure_log, *failure_args)
|
||||
return None, message
|
||||
return _CompletedPathResolution(
|
||||
None,
|
||||
message,
|
||||
retryable=_is_retryable_completed_path_probe(remapped_error),
|
||||
)
|
||||
|
||||
source_exists, source_error = _probe_completed_path(source_path_obj)
|
||||
|
||||
@@ -522,7 +594,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
download_id,
|
||||
source_path_obj,
|
||||
)
|
||||
return source_path_obj, None
|
||||
return _CompletedPathResolution(source_path_obj, None, retryable=False)
|
||||
|
||||
hint = _diagnose_path_issue(raw_path)
|
||||
if mappings:
|
||||
@@ -555,7 +627,12 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.error(failure_log, *failure_args)
|
||||
else:
|
||||
logger.debug(failure_log, *failure_args)
|
||||
return None, message
|
||||
return _CompletedPathResolution(
|
||||
None,
|
||||
message,
|
||||
retryable=not _path_needs_mapping(raw_path)
|
||||
and _is_retryable_completed_path_probe(source_error),
|
||||
)
|
||||
|
||||
def _wait_for_completed_path(
|
||||
self,
|
||||
@@ -567,23 +644,37 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
) -> tuple[Path | None, str | None]:
|
||||
"""Wait briefly for completed files to appear on disk."""
|
||||
last_error: str | None = None
|
||||
max_attempts = self._completed_path_max_attempts()
|
||||
retry_interval = self._completed_path_retry_interval()
|
||||
timeout_seconds = self._completed_path_timeout_seconds()
|
||||
if retry_interval <= 0 or timeout_seconds <= 0:
|
||||
max_attempts = 1
|
||||
else:
|
||||
max_attempts = int(math.ceil(timeout_seconds / retry_interval)) + 1
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
if cancel_flag and cancel_flag.is_set():
|
||||
return None, last_error
|
||||
|
||||
log_details = attempt == max_attempts
|
||||
resolved_path, error = self._resolve_download_path_once(
|
||||
result = self._resolve_download_path_once_detailed(
|
||||
client,
|
||||
download_id,
|
||||
log_details=log_details,
|
||||
)
|
||||
if resolved_path:
|
||||
return resolved_path, None
|
||||
if result.path:
|
||||
return result.path, None
|
||||
|
||||
last_error = error
|
||||
last_error = result.error
|
||||
|
||||
if not result.retryable:
|
||||
if not log_details:
|
||||
logger.error(
|
||||
"Completed path resolution is not retryable for %s (%s): %s",
|
||||
client.name,
|
||||
download_id,
|
||||
last_error,
|
||||
)
|
||||
return None, last_error
|
||||
|
||||
if attempt < max_attempts:
|
||||
status_callback("locating", "Waiting for completed files...")
|
||||
|
||||
@@ -258,7 +258,7 @@ class QBittorrentClient(DownloadClient):
|
||||
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
|
||||
|
||||
def _get_torrents_info(
|
||||
self, torrent_hash: str | None = None
|
||||
self, torrent_hash: str | None = None, category: str | None = None
|
||||
) -> tuple[list[SimpleNamespace], str | None]:
|
||||
"""Get torrent info using GET.
|
||||
|
||||
@@ -267,6 +267,7 @@ class QBittorrentClient(DownloadClient):
|
||||
- Keep "API/auth/connect" errors distinct from "torrent missing".
|
||||
- If a hash-specific query returns empty, fall back to listing by category
|
||||
and matching locally.
|
||||
- Without a hash, `category` narrows the listing to that category.
|
||||
|
||||
Returns:
|
||||
(torrents, error_message)
|
||||
@@ -301,6 +302,8 @@ class QBittorrentClient(DownloadClient):
|
||||
primary_params: dict[str, str] = {}
|
||||
if torrent_hash:
|
||||
primary_params["hashes"] = torrent_hash
|
||||
elif category:
|
||||
primary_params["category"] = category
|
||||
|
||||
response = do_request(primary_params)
|
||||
torrents, error = parse_response(response, request_params=primary_params)
|
||||
@@ -358,6 +361,44 @@ class QBittorrentClient(DownloadClient):
|
||||
else:
|
||||
return torrents, None
|
||||
|
||||
def _list_category_hashes(self, category: str | None) -> set[str] | None:
|
||||
"""Snapshot the hashes qBittorrent currently reports for a category."""
|
||||
torrents, error = self._get_torrents_info(category=category)
|
||||
if error:
|
||||
logger.debug("Could not snapshot qBittorrent torrents: %s", error)
|
||||
return None
|
||||
return {str(torrent.hash).lower() for torrent in torrents if getattr(torrent, "hash", None)}
|
||||
|
||||
def _discover_added_torrent_hash(
|
||||
self,
|
||||
name: str,
|
||||
category: str | None,
|
||||
known_hashes: set[str] | None,
|
||||
) -> str | None:
|
||||
"""Recover the hash of a torrent that was added without a known info_hash.
|
||||
|
||||
A `known_hashes` of None means the pre-add snapshot failed, so only a
|
||||
torrent matching the requested rename can identify the new arrival.
|
||||
"""
|
||||
for _ in range(20):
|
||||
torrents, error = self._get_torrents_info(category=category)
|
||||
if error:
|
||||
logger.debug("qBittorrent hash discovery: %s", error)
|
||||
else:
|
||||
new_torrents = [
|
||||
torrent
|
||||
for torrent in torrents
|
||||
if getattr(torrent, "hash", None)
|
||||
and (known_hashes is None or str(torrent.hash).lower() not in known_hashes)
|
||||
]
|
||||
for torrent in new_torrents:
|
||||
if getattr(torrent, "name", None) == name:
|
||||
return str(torrent.hash).lower()
|
||||
if known_hashes is not None and len(new_torrents) == 1:
|
||||
return str(new_torrents[0].hash).lower()
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if qBittorrent is configured and selected as the torrent client."""
|
||||
@@ -425,6 +466,10 @@ class QBittorrentClient(DownloadClient):
|
||||
expected_hash = torrent_info.info_hash
|
||||
torrent_data = torrent_info.torrent_data
|
||||
|
||||
known_hashes: set[str] | None = None
|
||||
if not expected_hash:
|
||||
known_hashes = self._list_category_hashes(category)
|
||||
|
||||
# Per-torrent seeding limits from indexer
|
||||
seeding_time_limit_value = kwargs.get("seeding_time_limit")
|
||||
seeding_time_limit = coerce_optional_int(seeding_time_limit_value)
|
||||
@@ -459,12 +504,17 @@ class QBittorrentClient(DownloadClient):
|
||||
result_text = _normalize_add_result(result)
|
||||
logger.debug("qBittorrent add result: %s", result_text)
|
||||
|
||||
if not expected_hash:
|
||||
_raise_runtime_error("Could not determine torrent hash from URL")
|
||||
|
||||
if _is_explicit_add_failure(result):
|
||||
_raise_runtime_error(f"Failed to add torrent: {result_text}")
|
||||
|
||||
if not expected_hash:
|
||||
# qBittorrent fetches .torrent URLs itself, so the add can succeed
|
||||
# even when no hash could be extracted up front. Recover it by
|
||||
# watching for the new torrent to appear.
|
||||
expected_hash = self._discover_added_torrent_hash(name, category, known_hashes)
|
||||
if not expected_hash:
|
||||
_raise_runtime_error("Could not determine torrent hash from URL")
|
||||
|
||||
# Some qBittorrent-compatible clients return HTTP 200 with an empty body
|
||||
# instead of qBittorrent's literal "Ok." response. Prefer verifying that
|
||||
# the torrent becomes visible over trusting the response body alone.
|
||||
|
||||
@@ -4,6 +4,7 @@ Uses xmlrpc to communicate with rTorrent's RPC interface.
|
||||
"""
|
||||
|
||||
import ssl
|
||||
import time
|
||||
import xmlrpc.client as stdlib_xmlrpc_client
|
||||
from typing import Any, NoReturn, Protocol, cast
|
||||
from urllib.parse import urlparse
|
||||
@@ -160,6 +161,10 @@ class RTorrentClient(DownloadClient):
|
||||
try:
|
||||
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
|
||||
known_hashes: set[str] | None = None
|
||||
if not (torrent_info.info_hash or expected_hash):
|
||||
known_hashes = self._list_torrent_hashes()
|
||||
|
||||
commands = []
|
||||
|
||||
is_audiobook = kwargs.get("content_type") == "audiobook"
|
||||
@@ -195,6 +200,11 @@ class RTorrentClient(DownloadClient):
|
||||
self._rpc.load.start("", add_url, ";".join(commands))
|
||||
|
||||
torrent_hash = torrent_info.info_hash or expected_hash
|
||||
if not torrent_hash:
|
||||
# rTorrent fetches .torrent URLs itself, so the add can succeed
|
||||
# even when no hash could be extracted up front. Recover it by
|
||||
# watching for the new download to appear.
|
||||
torrent_hash = self._discover_added_torrent_hash(name, label, known_hashes)
|
||||
if not torrent_hash:
|
||||
_raise_runtime_error("Could not determine torrent hash from URL")
|
||||
|
||||
@@ -387,6 +397,56 @@ class RTorrentClient(DownloadClient):
|
||||
except _RTORRENT_CLIENT_ERRORS:
|
||||
return "/downloads"
|
||||
|
||||
def _list_torrent_hashes(self) -> set[str] | None:
|
||||
"""Snapshot the hashes rTorrent currently reports."""
|
||||
try:
|
||||
all_torrents = self._rpc.d.multicall2("", "", "d.hash=")
|
||||
except _RTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("Could not snapshot rTorrent downloads: %s", e)
|
||||
return None
|
||||
return {str(row[0]).lower() for row in all_torrents if row and row[0]}
|
||||
|
||||
def _discover_added_torrent_hash(
|
||||
self,
|
||||
name: str,
|
||||
label: str,
|
||||
known_hashes: set[str] | None,
|
||||
) -> str | None:
|
||||
"""Recover the hash of a torrent that was added without a known info_hash.
|
||||
|
||||
rTorrent fetches .torrent URLs itself, so the add can succeed even when
|
||||
no hash could be extracted up front. A `known_hashes` of None means the
|
||||
pre-add snapshot failed, so only an exact name match can identify the
|
||||
new arrival.
|
||||
"""
|
||||
for _ in range(20):
|
||||
try:
|
||||
all_torrents = self._rpc.d.multicall2("", "", "d.hash=", "d.name=", "d.custom1=")
|
||||
except _RTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("rTorrent hash discovery: %s", e)
|
||||
else:
|
||||
new_torrents = [
|
||||
row
|
||||
for row in all_torrents
|
||||
if row
|
||||
and row[0]
|
||||
and (known_hashes is None or str(row[0]).lower() not in known_hashes)
|
||||
]
|
||||
# The label set at add time distinguishes concurrent arrivals,
|
||||
# but rTorrent may not have applied it yet, so it only ever
|
||||
# narrows a non-empty candidate list.
|
||||
if label:
|
||||
labeled = [row for row in new_torrents if len(row) > 2 and row[2] == label]
|
||||
if labeled:
|
||||
new_torrents = labeled
|
||||
for row in new_torrents:
|
||||
if len(row) > 1 and row[1] == name:
|
||||
return str(row[0]).lower()
|
||||
if known_hashes is not None and len(new_torrents) == 1:
|
||||
return str(new_torrents[0][0]).lower()
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
|
||||
def _get_torrent_path(self, download_id: str) -> str | None:
|
||||
"""Get the file path of a torrent by hash.
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from shelfmark.download.network import get_ssl_verify
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
_MAGNET_RESPONSE_MAX_BYTES = 2000
|
||||
_TORRENT_FETCH_MAX_REDIRECTS = 5
|
||||
_BASE32_BTMH_TAG_BYTES = 34
|
||||
_BTIH_INFO_BYTE_HEX = 0x20
|
||||
_BTIH_PREFIX_BYTE = 0x12
|
||||
@@ -98,19 +99,18 @@ def extract_torrent_info(
|
||||
|
||||
# A release source can legitimately hand us a download URL on a different
|
||||
# origin than the configured Prowlarr/Newznab endpoint (e.g. a direct
|
||||
# tracker link, or Prowlarr reached through a separate proxy). We still need
|
||||
# to fetch the .torrent to recover the info_hash when the source did not
|
||||
# provide one, so the prefetch runs regardless of origin. The Prowlarr API
|
||||
# key, however, is only ever sent to a trusted origin so it can never leak
|
||||
# to an arbitrary indexer/tracker host.
|
||||
trusted_origin = _is_trusted_torrent_fetch_url(url)
|
||||
|
||||
# tracker link, or Prowlarr reached through a separate proxy), and a trusted
|
||||
# Prowlarr download URL commonly redirects to the indexer's own download
|
||||
# link. We still need to fetch the .torrent to recover the info_hash when
|
||||
# the source did not provide one, so the prefetch runs regardless of origin
|
||||
# and follows cross-origin redirects. The Prowlarr API key, however, is
|
||||
# re-evaluated per hop and only ever sent to a trusted origin so it can
|
||||
# never leak to an arbitrary indexer/tracker host.
|
||||
headers: dict[str, str] = {"Accept": "application/x-bittorrent"}
|
||||
if trusted_origin:
|
||||
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
|
||||
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
|
||||
if api_key:
|
||||
headers["X-Api-Key"] = api_key
|
||||
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
|
||||
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
|
||||
if api_key:
|
||||
headers["X-Api-Key"] = api_key
|
||||
|
||||
def resolve_url(current: str, location: str) -> str:
|
||||
if not location:
|
||||
@@ -121,19 +121,28 @@ def extract_torrent_info(
|
||||
try:
|
||||
logger.debug("Fetching torrent file from: %s...", url[:80])
|
||||
|
||||
# Use allow_redirects=False to handle magnet link redirects manually
|
||||
# Some indexers redirect download URLs to magnet links
|
||||
resp = requests.get(
|
||||
url,
|
||||
timeout=30,
|
||||
allow_redirects=False,
|
||||
headers=headers,
|
||||
verify=get_ssl_verify(url),
|
||||
)
|
||||
# Redirects are followed manually: some indexers redirect download URLs
|
||||
# to magnet links, and each hop must decide anew whether it may see the
|
||||
# API key.
|
||||
current_url = url
|
||||
redirects_remaining = _TORRENT_FETCH_MAX_REDIRECTS
|
||||
while True:
|
||||
request_headers = dict(headers)
|
||||
if not _is_trusted_torrent_fetch_url(current_url):
|
||||
request_headers.pop("X-Api-Key", None)
|
||||
|
||||
# Check if this is a redirect to a magnet link
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
redirect_url = resolve_url(url, resp.headers.get("Location", ""))
|
||||
resp = requests.get(
|
||||
current_url,
|
||||
timeout=30,
|
||||
allow_redirects=False,
|
||||
headers=request_headers,
|
||||
verify=get_ssl_verify(current_url),
|
||||
)
|
||||
|
||||
if resp.status_code not in (301, 302, 303, 307, 308):
|
||||
break
|
||||
|
||||
redirect_url = resolve_url(current_url, resp.headers.get("Location", ""))
|
||||
if redirect_url.startswith("magnet:"):
|
||||
logger.debug("Download URL redirected to magnet link")
|
||||
info_hash = extract_hash_from_magnet(redirect_url)
|
||||
@@ -145,20 +154,12 @@ def extract_torrent_info(
|
||||
is_magnet=True,
|
||||
magnet_url=redirect_url,
|
||||
)
|
||||
if not _is_trusted_torrent_fetch_url(redirect_url):
|
||||
logger.debug(
|
||||
"Skipping torrent prefetch redirect to untrusted URL: %s...",
|
||||
redirect_url[:80],
|
||||
)
|
||||
if redirects_remaining <= 0:
|
||||
logger.debug("Too many redirects fetching torrent file: %s...", url[:80])
|
||||
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
|
||||
# Not a magnet redirect, follow it manually
|
||||
redirects_remaining -= 1
|
||||
logger.debug("Following redirect to: %s...", redirect_url[:80])
|
||||
resp = requests.get(
|
||||
redirect_url,
|
||||
timeout=30,
|
||||
headers=headers,
|
||||
verify=get_ssl_verify(redirect_url),
|
||||
)
|
||||
current_url = redirect_url
|
||||
|
||||
resp.raise_for_status()
|
||||
torrent_data = resp.content
|
||||
|
||||
@@ -852,6 +852,99 @@ def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver:
|
||||
return custom_resolver
|
||||
|
||||
|
||||
# --- ISP / network DNS interference detection ---------------------------------
|
||||
# Compare what the (tamperable) system resolver returns for a host against a
|
||||
# tamper-resistant DoH lookup. Divergent answers are a strong signal the network is
|
||||
# hijacking or NXDOMAIN-blocking the domain (a common reason AA downloads "work" but
|
||||
# land on an ISP block page). Used to surface an actionable hint to the user.
|
||||
_dns_interference_warned: set[str] = set()
|
||||
_dns_interference_active = False
|
||||
|
||||
|
||||
def _build_detection_doh_resolver() -> DoHResolver | None:
|
||||
"""Build a throwaway DoH resolver for interference checks (no socket patching).
|
||||
|
||||
Honours the DoH provider the user selected (``DNS_PROVIDERS[_current_dns_index]``),
|
||||
falling back to the first configured provider when none is active. The endpoint is
|
||||
pinned to the provider's own nameserver IP so resolving the DoH host can't be
|
||||
redirected by the very DNS layer the check is meant to detect.
|
||||
"""
|
||||
if 0 <= _current_dns_index < len(DNS_PROVIDERS):
|
||||
_name, servers, doh_url = DNS_PROVIDERS[_current_dns_index]
|
||||
elif DNS_PROVIDERS:
|
||||
_name, servers, doh_url = DNS_PROVIDERS[0]
|
||||
else:
|
||||
return None
|
||||
server_hostname = urllib.parse.urlparse(doh_url).hostname or ""
|
||||
if not server_hostname or not servers:
|
||||
return None
|
||||
return DoHResolver(doh_url, server_hostname, servers[0])
|
||||
|
||||
|
||||
def detect_dns_interference(hostname: str) -> dict[str, list[str]] | None:
|
||||
"""Detect network DNS interference by comparing system DNS against DoH.
|
||||
|
||||
Returns ``{"system_ips": [...], "doh_ips": [...]}`` when the two resolvers disagree
|
||||
(no overlapping IPs), otherwise None. No-op for IP literals / local hostnames and
|
||||
when DoH resolution is unavailable, so it never produces a false positive.
|
||||
"""
|
||||
host = (hostname or "").strip().lower()
|
||||
if not host or _is_ip_address(host) or _is_local_address(host):
|
||||
return None
|
||||
resolver = _build_detection_doh_resolver()
|
||||
if resolver is None:
|
||||
return None
|
||||
try:
|
||||
system_ips = {str(info[4][0]) for info in original_getaddrinfo(host, 443, socket.AF_INET)}
|
||||
except OSError:
|
||||
return None
|
||||
if not system_ips:
|
||||
return None
|
||||
doh_ips = {ip for ip in resolver.resolve(host, "A") if ip}
|
||||
if not doh_ips or (system_ips & doh_ips):
|
||||
return None
|
||||
return {"system_ips": sorted(system_ips), "doh_ips": sorted(doh_ips)}
|
||||
|
||||
|
||||
def note_possible_dns_interference(hostname: str) -> bool:
|
||||
"""Check ``hostname`` for DNS interference, logging an actionable warning once.
|
||||
|
||||
Returns True when interference has been detected this session. The check runs at
|
||||
most once per host to avoid repeated DoH lookups and log spam.
|
||||
"""
|
||||
global _dns_interference_active
|
||||
host = (hostname or "").strip().lower()
|
||||
if not host or host in _dns_interference_warned:
|
||||
return _dns_interference_active
|
||||
_dns_interference_warned.add(host)
|
||||
|
||||
result = detect_dns_interference(host)
|
||||
if not result:
|
||||
return _dns_interference_active
|
||||
|
||||
_dns_interference_active = True
|
||||
routing_via_doh = _current_dns_index >= 0 and bool(DOH_SERVER)
|
||||
remedy = (
|
||||
"Shelfmark is routing this domain through DNS-over-HTTPS to work around it."
|
||||
if routing_via_doh
|
||||
else "Enable DNS-over-HTTPS (USE_DOH=true) or set a custom DNS provider to bypass it."
|
||||
)
|
||||
logger.warning(
|
||||
"Possible ISP/network DNS interference for %s: system DNS resolves to %s but DoH "
|
||||
"resolves to %s. The network appears to be blocking or redirecting this domain. %s",
|
||||
host,
|
||||
result["system_ips"],
|
||||
result["doh_ips"],
|
||||
remedy,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def dns_interference_detected() -> bool:
|
||||
"""Whether network DNS interference has been detected this session."""
|
||||
return _dns_interference_active
|
||||
|
||||
|
||||
def init_doh_resolver(doh_server: str = "") -> DoHResolver | None:
|
||||
"""Initialize DNS over HTTPS resolver."""
|
||||
server = doh_server or DOH_SERVER
|
||||
|
||||
@@ -1621,6 +1621,12 @@ def _extract_slow_download_url(
|
||||
|
||||
link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
|
||||
logger.warning("No download URL found. First 10 links: %s", link_texts)
|
||||
# A bypassed page with no AA download links often means the network served a wrong
|
||||
# page (e.g. an ISP block page) instead of Anna's Archive. Probe for DNS interference
|
||||
# so we can give the user an actionable hint instead of a generic failure.
|
||||
host = urlparse(link).hostname or ""
|
||||
if host:
|
||||
network.note_possible_dns_interference(host)
|
||||
return ""
|
||||
|
||||
|
||||
@@ -2053,7 +2059,14 @@ class DirectDownloadHandler(DownloadHandler):
|
||||
return None
|
||||
|
||||
if not success_url:
|
||||
status_callback("error", "All download sources failed")
|
||||
if network.dns_interference_detected():
|
||||
status_callback(
|
||||
"error",
|
||||
"All sources failed - your network/ISP appears to be blocking "
|
||||
"Anna's Archive. Enable DNS-over-HTTPS in settings.",
|
||||
)
|
||||
else:
|
||||
status_callback("error", "All download sources failed")
|
||||
return None
|
||||
|
||||
# Return temp path - orchestrator handles post-processing (archive extraction, ingest)
|
||||
|
||||
Generated
+737
-578
File diff suppressed because it is too large
Load Diff
@@ -17,24 +17,24 @@
|
||||
"test:unit": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tailwindcss/vite": "^4.3.0",
|
||||
"@tailwindcss/vite": "^4.3.2",
|
||||
"react": "^19.2.7",
|
||||
"react-dom": "^19.2.7",
|
||||
"react-router-dom": "^7.17.0",
|
||||
"react-router-dom": "^7.18.1",
|
||||
"socket.io-client": "^4.7.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.9.3",
|
||||
"@types/node": "^26.0.1",
|
||||
"@types/react": "^19.2.17",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"knip": "^6.16.1",
|
||||
"oxfmt": "^0.54.0",
|
||||
"oxlint": "^1.69.0",
|
||||
"@vitejs/plugin-react": "^6.0.3",
|
||||
"knip": "^6.23.0",
|
||||
"oxfmt": "^0.57.0",
|
||||
"oxlint": "^1.72.0",
|
||||
"oxlint-tsgolint": "^0.23.0",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^6.0.3",
|
||||
"vite": "^8.0.16",
|
||||
"vitest": "^4.1.8"
|
||||
"vite": "^8.1.0",
|
||||
"vitest": "^4.1.9"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ export const CardView = ({
|
||||
};
|
||||
|
||||
return (
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
|
||||
<article
|
||||
className="book-card space-between animate-pop-up relative flex h-full w-full flex-col transition-shadow duration-300 max-sm:h-[180px] max-sm:flex-row sm:max-w-[292px] sm:flex-col"
|
||||
style={{
|
||||
|
||||
@@ -70,6 +70,7 @@ export const CompactView = ({
|
||||
};
|
||||
|
||||
return (
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
|
||||
<article
|
||||
className="book-card animate-pop-up relative flex! h-[180px]! w-full flex-row! transition-shadow duration-300"
|
||||
style={{
|
||||
|
||||
@@ -188,6 +188,7 @@ export function Tooltip({
|
||||
isVisible &&
|
||||
coords &&
|
||||
createPortal(
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- interactive tooltip stays open while hovered; keyboard users are served by focus handlers on the trigger element
|
||||
<div
|
||||
ref={tooltipRef}
|
||||
role="tooltip"
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Cluster 7 (audiobook/ABB) parse-contract guards.
|
||||
|
||||
ABB forces ``https://`` for search and detail fetches, so it can't be exercised
|
||||
hermetically in the HTTP e2e docker platform. Its recurring bugs are instead in
|
||||
*parsing*: magnet/info-hash extraction ("Fix ABB magnet parsing", and the
|
||||
qbittorrent hash-length issue #386) and DOM/layout drift. These contract tests
|
||||
feed golden HTML through the real scraper — the same fail-on-drift philosophy as
|
||||
the AA layout-drift guard — and run in normal CI.
|
||||
|
||||
They deliberately cover cases the existing ``test_scraper.py`` does not: info-hash
|
||||
*normalization* (whitespace/case), the in-page magnet *fallback*, and a layout
|
||||
drift that must degrade to an empty result rather than crash.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from unittest.mock import patch
|
||||
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
|
||||
# Detail page where the Info Hash is lowercase and split by whitespace/newlines —
|
||||
# the exact shape that produced malformed magnets / wrong hash lengths (#386).
|
||||
DETAIL_HTML_MESSY_HASH = """
|
||||
<html><body><table>
|
||||
<tr><td>Info Hash</td><td>abc123def456789012345678
|
||||
901234567890abcd</td></tr>
|
||||
<tr><td>Tracker 1</td><td>udp://tracker.openbittorrent.com:80</td></tr>
|
||||
</table></body></html>
|
||||
"""
|
||||
|
||||
# Info Hash cell is junk, but a full magnet link is posted elsewhere on the page.
|
||||
DETAIL_HTML_MAGNET_FALLBACK = """
|
||||
<html><body>
|
||||
<table><tr><td>Info Hash</td><td>n/a</td></tr></table>
|
||||
<p>Mirror: magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=x</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# DOM drift: results are present but the .post / .postTitle structure changed.
|
||||
SEARCH_HTML_LAYOUT_DRIFT = """
|
||||
<html><body>
|
||||
<article class="result-card">
|
||||
<header><a href="/abss/drifted/">Drifted Audiobook - Author</a></header>
|
||||
<span class="lang">English</span>
|
||||
</article>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
def _patch_detail(html: str):
|
||||
return patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
|
||||
return_value=html,
|
||||
)
|
||||
|
||||
|
||||
def test_info_hash_is_normalized_to_canonical_btih() -> None:
|
||||
"""Whitespace/newlines are stripped and the hash upper-cased to a valid
|
||||
40-char btih (regression for #386 / 'Fix ABB magnet parsing')."""
|
||||
with _patch_detail(DETAIL_HTML_MESSY_HASH):
|
||||
magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/x/", "audiobookbay.lu")
|
||||
assert magnet is not None, "messy-but-valid info hash should still yield a magnet"
|
||||
btih = re.search(r"xt=urn:btih:([0-9A-Fa-f]+)", magnet)
|
||||
assert btih is not None, magnet
|
||||
assert btih.group(1) == "ABC123DEF456789012345678901234567890ABCD"
|
||||
assert len(btih.group(1)) == 40
|
||||
assert "tr=" in magnet # tracker carried through
|
||||
|
||||
|
||||
def test_magnet_fallback_when_info_hash_cell_is_junk() -> None:
|
||||
"""When the Info Hash cell is invalid, the scraper recovers the hash from an
|
||||
in-page magnet link rather than failing."""
|
||||
with _patch_detail(DETAIL_HTML_MAGNET_FALLBACK):
|
||||
magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/y/", "audiobookbay.lu")
|
||||
assert magnet is not None
|
||||
assert "btih:1111111111111111111111111111111111111111" in magnet
|
||||
|
||||
|
||||
def test_missing_info_hash_returns_none_not_crash() -> None:
|
||||
"""No hash anywhere -> None (clean failure), never an exception."""
|
||||
with _patch_detail("<html><body><p>nothing here</p></body></html>"):
|
||||
assert (
|
||||
scraper.extract_magnet_link("https://audiobookbay.lu/abss/z/", "audiobookbay.lu")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_search_layout_drift_degrades_to_empty() -> None:
|
||||
"""A changed results DOM yields zero parsed results without raising — the
|
||||
ABB analogue of the AA layout-drift guard."""
|
||||
with (
|
||||
patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
|
||||
return_value=(SEARCH_HTML_LAYOUT_DRIFT, "https://audiobookbay.lu/?s=test"),
|
||||
),
|
||||
patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.config.get",
|
||||
return_value=0.0,
|
||||
),
|
||||
):
|
||||
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
|
||||
assert results == [], f"drifted DOM should parse to no results, got {results!r}"
|
||||
@@ -274,6 +274,75 @@ def test_run_child_process_writes_failure_for_unexpected_exception(monkeypatch,
|
||||
assert "plain SeleniumBase startup failure" in result["traceback"]
|
||||
|
||||
|
||||
def test_run_child_process_applies_parent_dns_config(monkeypatch, tmp_path):
|
||||
"""Regression test for issue #1028: the helper subprocess must mirror the parent's
|
||||
DNS provider, otherwise it pre-resolves AA hostnames against (possibly hijacked)
|
||||
system DNS and Chrome loads the wrong page."""
|
||||
import io
|
||||
import json
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
result_path = tmp_path / "result.json"
|
||||
request = {
|
||||
"url": "https://annas-archive.pk/slow_download/abc/0/0",
|
||||
"retry": 1,
|
||||
"result_path": str(result_path),
|
||||
"dns_config": {
|
||||
"provider": "cloudflare",
|
||||
"servers": ["1.1.1.1", "1.0.0.1"],
|
||||
"doh_url": "https://cloudflare-dns.com/dns-query",
|
||||
"doh_enabled": True,
|
||||
"is_auto_mode": True,
|
||||
},
|
||||
}
|
||||
|
||||
applied: list[tuple] = []
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.network,
|
||||
"set_dns_provider",
|
||||
lambda provider, manual=None, *, use_doh=None: applied.append((provider, manual, use_doh)),
|
||||
)
|
||||
monkeypatch.setattr(internal_bypasser, "get", lambda *_a, **_k: "<html>ok</html>")
|
||||
monkeypatch.setattr(internal_bypasser.sys, "stdin", io.StringIO(json.dumps(request)))
|
||||
|
||||
assert internal_bypasser._run_child_process() == 0
|
||||
assert applied == [("cloudflare", None, True)]
|
||||
|
||||
|
||||
def test_apply_parent_dns_config_skips_auto_and_empty(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
calls: list = []
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.network,
|
||||
"set_dns_provider",
|
||||
lambda *a, **k: calls.append((a, k)),
|
||||
)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config({"provider": "auto"})
|
||||
internal_bypasser._apply_parent_dns_config({})
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_apply_parent_dns_config_forwards_manual_servers(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
calls: list = []
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.network,
|
||||
"set_dns_provider",
|
||||
lambda provider, manual=None, *, use_doh=None: calls.append((provider, manual, use_doh)),
|
||||
)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "manual", "servers": ["9.9.9.9"], "doh_enabled": False}
|
||||
)
|
||||
|
||||
assert calls == [("manual", ["9.9.9.9"], False)]
|
||||
|
||||
|
||||
def test_prepare_child_browser_env_uses_writable_runtime_paths(monkeypatch, tmp_path):
|
||||
import stat
|
||||
|
||||
|
||||
@@ -257,3 +257,119 @@ def test_entrypoint_root_bootstrap_fails_closed_when_config_repair_fails(tmp_pat
|
||||
assert f"Configured runtime identity: {os.getuid()}:{os.getgid()}" in result.stdout
|
||||
assert f"chown -R {os.getuid()}:{os.getgid()} /path/to/config" in result.stdout
|
||||
assert "Startup mode: root" not in result.stdout
|
||||
|
||||
|
||||
def test_entrypoint_rejects_wireguard_in_non_root_mode(tmp_path):
|
||||
result, _, _, _ = _run_entrypoint(tmp_path, extra_env={"USING_WIREGUARD": "true"})
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "USING_WIREGUARD=true requires the container to start as root." in result.stderr
|
||||
assert "Non-root mode skips the privileged network setup WireGuard depends on." in result.stderr
|
||||
|
||||
|
||||
def test_entrypoint_rejects_tor_and_wireguard_together(tmp_path):
|
||||
result, _, _, _ = _run_entrypoint(
|
||||
tmp_path,
|
||||
extra_env={"USING_TOR": "true", "USING_WIREGUARD": "true"},
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert (
|
||||
"USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode."
|
||||
in result.stderr
|
||||
)
|
||||
# The mutual-exclusion check must fire before either egress script runs, so
|
||||
# neither the Tor nor the WireGuard privileged-setup errors should appear.
|
||||
assert "requires the container to start as root" not in result.stderr
|
||||
|
||||
|
||||
def test_entrypoint_mutual_exclusion_precedes_tor_startup(tmp_path):
|
||||
# Even in root mode, enabling both must fail fast on mutual exclusion rather
|
||||
# than starting tor.sh and then aborting.
|
||||
result, _, _, _ = _run_entrypoint(
|
||||
tmp_path,
|
||||
simulate_root_startup=True,
|
||||
extra_env={"USING_TOR": "true", "USING_WIREGUARD": "true"},
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert (
|
||||
"USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode."
|
||||
in result.stderr
|
||||
)
|
||||
|
||||
|
||||
def test_entrypoint_aborts_before_gunicorn_when_wireguard_fails(tmp_path):
|
||||
"""Security invariant: if wireguard.sh exits non-zero (any fail-closed path),
|
||||
entrypoint.sh must abort under `set -e` so gunicorn NEVER starts. A booting
|
||||
app after a failed egress setup would be a kill-switch bypass / IP leak.
|
||||
"""
|
||||
# Run the REAL entrypoint from a temp cwd that provides a stub `./wireguard.sh`
|
||||
# which exits 1, plus a stub `./tor.sh` (unused here) for completeness.
|
||||
work = tmp_path / "work"
|
||||
work.mkdir()
|
||||
real_entrypoint = ENTRYPOINT_PATH.read_text()
|
||||
(work / "entrypoint.sh").write_text(real_entrypoint)
|
||||
(work / "entrypoint.sh").chmod(0o755)
|
||||
_write_executable(
|
||||
work / "wireguard.sh",
|
||||
"#!/bin/sh\necho 'stub wireguard.sh failing closed' >&2\nexit 1\n",
|
||||
)
|
||||
_write_executable(work / "tor.sh", "#!/bin/sh\nexit 0\n")
|
||||
|
||||
bin_dir, runtime_home_file, runtime_args_file = _build_stub_bin(tmp_path)
|
||||
config_dir = tmp_path / "config"
|
||||
config_dir.mkdir(exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"BUILD_VERSION": "test-build",
|
||||
"CONFIG_DIR": str(config_dir),
|
||||
"DEBUG": "false",
|
||||
"ENABLE_LOGGING": "false",
|
||||
"ENTRYPOINT_GUNICORN_ARGS_FILE": str(runtime_args_file),
|
||||
"ENTRYPOINT_GUNICORN_HOME_FILE": str(runtime_home_file),
|
||||
"ENTRYPOINT_REAL_ID": ID_PATH,
|
||||
"ENTRYPOINT_REAL_MKDIR": MKDIR_PATH,
|
||||
"ENTRYPOINT_REAL_STAT": STAT_PATH,
|
||||
"ENTRYPOINT_STUB_GID": str(os.getgid()),
|
||||
"ENTRYPOINT_STUB_HOME": str(tmp_path / "runtime-home"),
|
||||
"ENTRYPOINT_STUB_UID": str(os.getuid()),
|
||||
# Root startup so the WireGuard branch runs ./wireguard.sh (our stub).
|
||||
"ENTRYPOINT_STUB_CURRENT_GID": "0",
|
||||
"ENTRYPOINT_STUB_CURRENT_UID": "0",
|
||||
"ENTRYPOINT_STUB_STAT_OWNER": "0:0",
|
||||
"FLASK_PORT": "8084",
|
||||
"LOG_LEVEL": "info",
|
||||
"LOG_ROOT": str(tmp_path / "logs"),
|
||||
"PATH": f"{bin_dir}:{env.get('PATH', '')}",
|
||||
"PGID": str(os.getgid()),
|
||||
"PUID": str(os.getuid()),
|
||||
"RELEASE_VERSION": "test-release",
|
||||
"TMP_DIR": str(tmp_path / "tmp"),
|
||||
"TZ": "",
|
||||
"USING_EXTERNAL_BYPASSER": "true",
|
||||
"USING_WIREGUARD": "true",
|
||||
}
|
||||
)
|
||||
|
||||
with _entrypoint_lock():
|
||||
result = subprocess.run(
|
||||
[BASH_PATH, str(work / "entrypoint.sh")],
|
||||
capture_output=True,
|
||||
cwd=work,
|
||||
env=env,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
# Entrypoint must have aborted with the stub's non-zero status...
|
||||
assert result.returncode != 0
|
||||
# ...the failure must actually come from wireguard.sh (not some unrelated
|
||||
# earlier abort), proven by the stub's marker on stderr...
|
||||
assert "stub wireguard.sh failing closed" in result.stderr
|
||||
# ...and gunicorn must NEVER have been invoked (args file never written).
|
||||
assert not runtime_args_file.exists(), (
|
||||
"gunicorn was started despite wireguard.sh failing — kill-switch bypass!"
|
||||
)
|
||||
|
||||
@@ -507,6 +507,15 @@ class TestConcurrencyConfiguration:
|
||||
assert interval >= 1
|
||||
assert interval <= 10
|
||||
|
||||
def test_completed_path_timeout_default(self):
|
||||
"""Completed external-client path wait should default to the legacy grace period."""
|
||||
from shelfmark.core.config import config
|
||||
|
||||
config.refresh()
|
||||
|
||||
timeout = config.get("DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT", 60)
|
||||
assert timeout == 60
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Cache Configuration Tests
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""DoH resolver integration against the e2e platform's mock DoH responder.
|
||||
|
||||
The config-cluster analysis flagged DNS/DoH as a recurring break surface (#1028,
|
||||
#108). A fully hermetic DoH-over-the-network profile isn't feasible in the HTTP
|
||||
docker platform (DoH provider URLs are HTTPS + IP-pinned), so we exercise the
|
||||
*real* ``DoHResolver`` client against the platform's mock ``doh`` role here, over
|
||||
plain HTTP on localhost. This runs in normal CI (not just the nightly docker
|
||||
matrix) and guards the DoH JSON-parsing path the app relies on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from wsgiref.simple_server import WSGIServer, make_server
|
||||
|
||||
import pytest
|
||||
|
||||
MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py"
|
||||
|
||||
|
||||
def _load_mock_doh_app(doh_map: str):
|
||||
"""Import the platform mock_services module wired for the ``doh`` role.
|
||||
|
||||
The module wires its routes at import time from ``MOCK_ROLE``/``DOH_MAP``, so
|
||||
those must be set before loading it.
|
||||
"""
|
||||
os.environ["MOCK_ROLE"] = "doh"
|
||||
os.environ["DOH_MAP"] = doh_map
|
||||
spec = importlib.util.spec_from_file_location("mock_doh_services", MOCK_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.app
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def doh_url():
|
||||
if not MOCK_PATH.exists():
|
||||
pytest.skip(f"platform mock not found at {MOCK_PATH}")
|
||||
app = _load_mock_doh_app("aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11")
|
||||
server: WSGIServer = make_server("127.0.0.1", 0, app)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
# Google JSON DoH style uses the /resolve endpoint.
|
||||
yield f"http://127.0.0.1:{server.server_port}/resolve"
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def _resolver(doh_url: str):
|
||||
from shelfmark.download.network import DoHResolver
|
||||
|
||||
# hostname/ip args are the DoH server's own identity (used only for recursion
|
||||
# avoidance); the localhost values here are irrelevant to the lookups under test.
|
||||
return DoHResolver(doh_url, "127.0.0.1", "127.0.0.1")
|
||||
|
||||
|
||||
def test_doh_resolves_mapped_host(doh_url) -> None:
|
||||
"""The real DoH client parses the mock's JSON answer into an A record."""
|
||||
assert _resolver(doh_url).resolve("aa.mock.test", "A") == ["172.30.0.10"]
|
||||
|
||||
|
||||
def test_doh_nxdomain_returns_empty_not_error(doh_url) -> None:
|
||||
"""An unmapped name yields an empty list (Status 3), not an exception —
|
||||
the path that, when mishandled, surfaced as silent download failures."""
|
||||
assert _resolver(doh_url).resolve("unmapped.invalid", "A") == []
|
||||
|
||||
|
||||
def test_doh_resolver_caches_within_ttl(doh_url) -> None:
|
||||
"""A second lookup is served from cache (the resolver's documented behaviour)."""
|
||||
resolver = _resolver(doh_url)
|
||||
first = resolver.resolve("cf.mock.test", "A")
|
||||
assert first == ["172.30.0.11"]
|
||||
assert ("cf.mock.test", "A") in resolver._cache
|
||||
assert resolver.resolve("cf.mock.test", "A") == first
|
||||
@@ -137,3 +137,135 @@ def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch):
|
||||
assert result == [
|
||||
(network.socket.AF_INET, network.socket.SOCK_STREAM, 6, "", ("203.0.113.10", 443))
|
||||
]
|
||||
|
||||
|
||||
def _addrinfo(ip):
|
||||
return [(2, 1, 6, "", (ip, 443))]
|
||||
|
||||
|
||||
class _FakeDoHResolver:
|
||||
def __init__(self, ips):
|
||||
self._ips = ips
|
||||
|
||||
def resolve(self, _hostname, _record_type):
|
||||
return list(self._ips)
|
||||
|
||||
|
||||
def test_build_detection_doh_resolver_uses_selected_provider(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(
|
||||
network,
|
||||
"DNS_PROVIDERS",
|
||||
[
|
||||
("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
|
||||
("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(network, "_current_dns_index", 1) # user selected quad9
|
||||
|
||||
resolver = network._build_detection_doh_resolver()
|
||||
|
||||
assert resolver is not None
|
||||
assert resolver.base_url == "https://dns.quad9.net/dns-query"
|
||||
assert resolver.hostname == "dns.quad9.net"
|
||||
assert resolver.ip == "9.9.9.9"
|
||||
|
||||
|
||||
def test_build_detection_doh_resolver_falls_back_to_first_provider(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(
|
||||
network,
|
||||
"DNS_PROVIDERS",
|
||||
[
|
||||
("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
|
||||
("quad9", ["9.9.9.9"], "https://dns.quad9.net/dns-query"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(network, "_current_dns_index", -1) # system / not yet rotated
|
||||
|
||||
resolver = network._build_detection_doh_resolver()
|
||||
|
||||
assert resolver is not None
|
||||
assert resolver.base_url == "https://cloudflare-dns.com/dns-query"
|
||||
assert resolver.ip == "1.1.1.1"
|
||||
|
||||
|
||||
def test_detect_dns_interference_flags_divergent_resolvers(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
# System DNS (hijacked) returns an ISP block-page IP; DoH returns the real one.
|
||||
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
|
||||
monkeypatch.setattr(
|
||||
network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
|
||||
)
|
||||
|
||||
result = network.detect_dns_interference("annas-archive.pk")
|
||||
|
||||
assert result == {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
|
||||
|
||||
|
||||
def test_detect_dns_interference_none_when_resolvers_agree(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("203.0.113.7"))
|
||||
monkeypatch.setattr(
|
||||
network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
|
||||
)
|
||||
|
||||
assert network.detect_dns_interference("annas-archive.pk") is None
|
||||
|
||||
|
||||
def test_detect_dns_interference_none_when_doh_unavailable(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
|
||||
monkeypatch.setattr(network, "_build_detection_doh_resolver", lambda: None)
|
||||
|
||||
assert network.detect_dns_interference("annas-archive.pk") is None
|
||||
|
||||
|
||||
def test_detect_dns_interference_skips_ip_and_local(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
def _should_not_run():
|
||||
raise AssertionError("resolver should not be built for IP/local hosts")
|
||||
|
||||
monkeypatch.setattr(network, "_build_detection_doh_resolver", _should_not_run)
|
||||
|
||||
assert network.detect_dns_interference("1.2.3.4") is None
|
||||
assert network.detect_dns_interference("localhost") is None
|
||||
|
||||
|
||||
def test_note_possible_dns_interference_warns_once_and_sets_flag(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "_dns_interference_warned", set())
|
||||
monkeypatch.setattr(network, "_dns_interference_active", False)
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
network,
|
||||
"detect_dns_interference",
|
||||
lambda host: (
|
||||
calls.append(host) or {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
|
||||
),
|
||||
)
|
||||
|
||||
assert network.note_possible_dns_interference("annas-archive.pk") is True
|
||||
assert network.dns_interference_detected() is True
|
||||
# A repeat check for the same host must not re-run the costly detection.
|
||||
assert network.note_possible_dns_interference("annas-archive.pk") is True
|
||||
assert calls == ["annas-archive.pk"]
|
||||
|
||||
|
||||
def test_note_possible_dns_interference_no_detection_keeps_flag_false(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "_dns_interference_warned", set())
|
||||
monkeypatch.setattr(network, "_dns_interference_active", False)
|
||||
monkeypatch.setattr(network, "detect_dns_interference", lambda _host: None)
|
||||
|
||||
assert network.note_possible_dns_interference("annas-archive.pk") is False
|
||||
assert network.dns_interference_detected() is False
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Validate the e2e platform's webseed .torrent generator.
|
||||
|
||||
The ``full`` e2e profile relies on a tracker-less webseed torrent so a real
|
||||
qBittorrent can complete a real download from the mock origin over HTTP. If the
|
||||
generator emits malformed bencode or mismatched piece hashes, qBittorrent would
|
||||
silently never complete — so we cross-check the generator against shelfmark's own
|
||||
``bencode_decode`` / ``extract_info_hash_from_torrent`` here, in normal CI.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
bencode_decode,
|
||||
extract_info_hash_from_torrent,
|
||||
)
|
||||
|
||||
GEN_PATH = (
|
||||
Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "make_webseed_torrent.py"
|
||||
)
|
||||
|
||||
|
||||
def _load_generator():
|
||||
if not GEN_PATH.exists():
|
||||
pytest.skip(f"generator not found at {GEN_PATH}")
|
||||
spec = importlib.util.spec_from_file_location("make_webseed_torrent", GEN_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
PAYLOAD = b"E2E webseed payload \x00\x01\x02 " * 2000 # ~50 KiB -> multiple pieces
|
||||
NAME = "sample-book.epub"
|
||||
WEBSEED = "http://mock-aa/payload/sample-book.epub"
|
||||
|
||||
|
||||
def test_generated_torrent_decodes_with_shelfmark_bencode() -> None:
|
||||
gen = _load_generator()
|
||||
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=16384)
|
||||
|
||||
decoded, _ = bencode_decode(raw)
|
||||
assert isinstance(decoded, dict)
|
||||
info = decoded[b"info"]
|
||||
assert info[b"name"] == NAME.encode()
|
||||
assert info[b"length"] == len(PAYLOAD)
|
||||
# url-list (webseed) must point at the file the mock serves.
|
||||
assert decoded[b"url-list"] == [WEBSEED.encode()]
|
||||
# No tracker — the whole point is HTTP-only completion.
|
||||
assert b"announce" not in decoded
|
||||
|
||||
|
||||
def test_piece_hashes_match_payload_bytes() -> None:
|
||||
gen = _load_generator()
|
||||
piece_len = 16384
|
||||
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=piece_len)
|
||||
decoded, _ = bencode_decode(raw)
|
||||
pieces = decoded[b"info"][b"pieces"]
|
||||
|
||||
expected = b"".join(
|
||||
hashlib.sha1(PAYLOAD[i : i + piece_len]).digest() for i in range(0, len(PAYLOAD), piece_len)
|
||||
)
|
||||
assert pieces == expected, "piece hashes do not match payload — qbit would never complete"
|
||||
assert len(pieces) % 20 == 0
|
||||
|
||||
|
||||
def test_info_hash_matches_shelfmark_extractor() -> None:
|
||||
"""Our infohash helper must agree with shelfmark's torrent parser."""
|
||||
gen = _load_generator()
|
||||
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
|
||||
|
||||
ours = gen.info_hash(raw)
|
||||
theirs = extract_info_hash_from_torrent(raw)
|
||||
assert theirs is not None
|
||||
assert ours.lower() == theirs.lower(), (ours, theirs)
|
||||
|
||||
|
||||
def test_generator_is_deterministic() -> None:
|
||||
gen = _load_generator()
|
||||
a = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
|
||||
b = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
|
||||
assert a == b, "torrent generation must be byte-deterministic for stable infohash"
|
||||
@@ -0,0 +1,7 @@
|
||||
# Runtime state created by run-e2e.sh: per-profile shelfmark logs, the app's
|
||||
# /config (settings, users.db, secrets), downloaded books, and staging dirs.
|
||||
.state/
|
||||
|
||||
# Python bytecode from the mock services + suite.
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1,231 @@
|
||||
# Shelfmark E2E Docker Testing Platform
|
||||
|
||||
A hermetic, container-based end-to-end platform that boots the real Shelfmark app
|
||||
against **controllable** dependencies — a fake Anna's Archive, a Cloudflare gate, a
|
||||
mock FlareSolverr bypasser, a mock Prowlarr + a real qBittorrent, custom DNS
|
||||
servers, HTTP/SOCKS5 proxies, and a Tor profile — and runs a cluster test suite
|
||||
under each **config profile**.
|
||||
|
||||
It exists to make the recurring bug clusters from the issue/PR analysis impossible
|
||||
to reintroduce silently. The biggest one — Tor/Cloudflare/bypasser (37 issues /
|
||||
67 fix PRs) — had almost no automated coverage; this platform changes that.
|
||||
|
||||
```
|
||||
pytest suite (host :8084)
|
||||
│
|
||||
▼
|
||||
shelfmark (under test) ── egress depends on the active profile:
|
||||
├─ direct ──────────────► mock-aa (.10) fake Anna's Archive
|
||||
├─ Cloudflare gate ─────► mock-cf (.11) ─► mock-aa [full: real Chrome solves it]
|
||||
├─ FlareSolverr ────────► mock-cf (.11) ─► mock-flaresolverr (.12) [bypasser-external]
|
||||
├─ custom DNS ──────────► coredns (.20) / coredns-blocked (.22) [dns-manual/blocked]
|
||||
├─ HTTP / SOCKS proxy ──► tinyproxy (.30) / microsocks (.31) [proxy-http/socks]
|
||||
├─ Tor (transparent) ───► in-image tor.sh [tor]
|
||||
└─ Prowlarr → client ───► mock-prowlarr (.40) ─► qBittorrent [full: real download]
|
||||
|
||||
(all on one e2e docker network, 172.30.0.0/24, static IPs for DNS determinism)
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# one profile
|
||||
make e2e-platform # baseline
|
||||
make e2e-platform-profile PROFILE=bypasser-external
|
||||
make e2e-platform-profile PROFILE=client-deluge
|
||||
make e2e-platform-full # heavy: real Chrome + DoH + real qBittorrent
|
||||
|
||||
# the whole matrix
|
||||
make e2e-platform-matrix
|
||||
|
||||
# build the heavy image once, then reuse it (matrix does this automatically)
|
||||
make e2e-platform-build
|
||||
E2E_NO_BUILD=1 tests/e2e/platform/run-e2e.sh env/dns-doh.env
|
||||
|
||||
# debug: leave the stack up after the run
|
||||
KEEP_UP=1 tests/e2e/platform/run-e2e.sh env/dns-blocked.env
|
||||
```
|
||||
|
||||
Requirements: Docker + Compose v2, and `uv` (for the pytest runner). The runner
|
||||
builds the Shelfmark image from the repo `Dockerfile`, boots the profile's stack,
|
||||
waits for `/api/health`, runs the suite, and tears down. `run-matrix.sh` builds the
|
||||
image **once** and reuses it across profiles (`E2E_NO_BUILD=1`) so the slow
|
||||
xvfb/chromium layer isn't rebuilt per profile.
|
||||
|
||||
## How profiles work
|
||||
|
||||
Each profile is an env file in `env/`. It sets:
|
||||
- `COMPOSE_PROFILES` — which optional services start (compose `profiles:`).
|
||||
- `SM_*` — the app's config, injected as container env. Shelfmark treats
|
||||
deployment ENV as authoritative (`config.get`: "Deployment-level ENV values
|
||||
always win"), so a profile fully determines the app's DNS/proxy/bypasser/source
|
||||
configuration with no runtime mutation.
|
||||
- `E2E_PROFILE` — handed to pytest so the suite selects applicable tests.
|
||||
|
||||
Tests declare applicability with `@pytest.mark.profiles(...)`. **A test with no
|
||||
marker is a profile-agnostic invariant and runs under every profile** — that is
|
||||
how one cluster test ("source must be reachable") becomes the config matrix.
|
||||
|
||||
## The matrix (cluster × profile)
|
||||
|
||||
Status column: ✅ = run live on Docker and passing. Every profile below was run
|
||||
end-to-end (`docker compose up` + suite + teardown) and passes.
|
||||
|
||||
| Profile | Egress / what it proves | Clusters | Regression targets | Status |
|
||||
|---|---|---|---|---|
|
||||
| `baseline` | Direct to fake AA; search/parse + #1028 clean-failure | 2,3,4 | #198 #293 #214 #1040 #1028 | ✅ 9 passed |
|
||||
| `bypasser-external` | External bypasser wired; CF-gated search fails cleanly | 1 | #284 #202 #410 #369 | ✅ 5 passed |
|
||||
| `bypasser-disabled` | CF-gated AA + bypasser OFF → no results (control) | 1 | #202 #410 | ✅ 4 passed |
|
||||
| `dns-manual` | AA only resolvable via custom DNS (coredns) | config: DNS | #108 | ✅ 4 passed |
|
||||
| `dns-blocked` | System DNS NXDOMAINs AA; custom DNS resolves it | config: DNS | **#1028** | ✅ 4 passed |
|
||||
| `dns-doh` | System DNS blocks AA; **DoH over real HTTPS** resolves it | config: DoH | **#1028** #108 | ✅ 3 passed |
|
||||
| `proxy-http` | All egress via tinyproxy, **proven by proxy logs** | config: proxy | **#956** | ✅ 6 passed |
|
||||
| `proxy-socks` | All egress via SOCKS5 (microsocks), traversal-checked | config: proxy | #956 | ✅ 5 passed |
|
||||
| `tor` | `USING_TOR=true` boots clean (restarts=0) | 1/6 Tor boot | #1021 #940 #801 | ✅ 5 passed |
|
||||
| `client-transmission` | Prowlarr → **real Transmission** webseed download → /books | 5 clients | #1022 #634 | ✅ 4 passed |
|
||||
| `client-deluge` | Prowlarr → **real Deluge** webseed download → /books | 5 clients | #530 | ✅ 4 passed |
|
||||
| `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed |
|
||||
| *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ |
|
||||
|
||||
> **The bypasser is download-time, not search-time.** Running the stack revealed
|
||||
> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a
|
||||
> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser
|
||||
> (internal Chrome or external FlareSolverr) only runs during a file *download*
|
||||
> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean*
|
||||
> CF-gated-search failure, while the **`full` profile exercises the real end-to-end
|
||||
> CF solve**: AA search/detail are reachable, but the AA slow-download link points
|
||||
> at the gate, so downloading Moby-Dick forces the in-image headless Chromium to
|
||||
> detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file —
|
||||
> verified live (`Challenge detected: cloudflare` → `Bypass successful` → Moby-Dick
|
||||
> in `/books`).
|
||||
>
|
||||
> **`bypasser-external` must set `SM_USING_EXTERNAL_BYPASSER=true`** — shelfmark does
|
||||
> **not** derive it from `EXT_BYPASSER_URL`; without it the app silently uses the
|
||||
> in-image Chrome bypasser instead of FlareSolverr.
|
||||
|
||||
Coverage of the 7 clusters from the analysis:
|
||||
|
||||
1. **Bypasser/Tor/Cloudflare** → `bypasser-external`, `bypasser-disabled`, `tor`.
|
||||
2. **Search/metadata** → `baseline` (`test_cluster_search_aa.py`, hermetic via the
|
||||
`direct_download` source so no external metadata provider is needed).
|
||||
3. **AA parsing/mirrors** → `baseline` parse guards incl. the **layout-drift
|
||||
fail-loud** test (#878/#879/#880).
|
||||
4. **Permissions/file-move** → `baseline` (`test_cluster_download_permissions.py`).
|
||||
5. **Torrent/usenet clients** → a mock Prowlarr + webseed torrent drives **three
|
||||
real torrent clients** end to end (`full`=qBittorrent, `client-transmission`,
|
||||
`client-deluge`) — completion detection + file move into `/books`. One
|
||||
client-agnostic test (`test_cluster_clients.py`) covers all three.
|
||||
6. **Docker/entrypoint/PUID-PGID** → profile-agnostic health + boot-log checks,
|
||||
run under every profile.
|
||||
7. **Audiobook/ABB** → parse-contract guards in
|
||||
`tests/audiobookbay/test_scraper_contract.py` (info-hash normalization #386,
|
||||
magnet fallback, layout drift). These run in **normal CI**, not the docker
|
||||
matrix, because ABB hardcodes `https://` for its fetches (see Roadmap).
|
||||
|
||||
### Proxy traversal (not just reachability)
|
||||
Because the app and the mock AA share the e2e network, a regression that ignores
|
||||
the proxy config would still reach AA directly. `test_egress_actually_traverses_proxy`
|
||||
drives a search and then inspects the proxy container's logs, so the proxy
|
||||
profiles prove the egress *went through* the proxy — a real guard for #956.
|
||||
|
||||
### DoH — two layers
|
||||
- **Offline** (`tests/download/test_doh_resolver_mock.py`, normal CI): the real
|
||||
`DoHResolver` is driven against the mock `doh` role over localhost HTTP, covering
|
||||
JSON-answer parsing, NXDOMAIN → empty, and caching.
|
||||
- **In-stack** (`dns-doh` profile): the mock `doh` role serves the DNS JSON API over
|
||||
**real HTTPS** (self-signed). The system resolver (coredns-blocked) NXDOMAINs
|
||||
`aa.mock.test`, so the host can *only* be resolved via DoH; compose `extra_hosts`
|
||||
redirects the `cloudflare-dns.com` provider to the in-stack mock and
|
||||
`CERTIFICATE_VALIDATION=disabled` accepts the self-signed cert. The search reaching
|
||||
AA proves the app's DoH path resolved the name end to end — **no app code change**.
|
||||
|
||||
### The `full` profile — real Chrome + real client (`make e2e-platform-full`)
|
||||
The "everything real" heavy profile (test book: **Moby-Dick**), run nightly / on
|
||||
demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, and
|
||||
**passes live** (6 passed):
|
||||
|
||||
- **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are
|
||||
reachable (`mock-aa`), but the AA *slow-download* link points at the Cloudflare
|
||||
gate (`mock-cf`), whose challenge page runs JS that issues `cf_clearance` and
|
||||
reloads. Downloading Moby-Dick forces the in-image headless Chromium (seleniumbase
|
||||
CDP, in the `shelfmark` image via `xvfb`+`chromium`) to load the gate, detect the
|
||||
challenge (`Challenge detected: cloudflare`), solve it (`_bypass_method_cdp_solve`),
|
||||
and fetch the cleared "Download now" page → the file lands in `/books`. That
|
||||
outcome is *only* reachable if Chrome solved the gate — the literal "spin a Chrome
|
||||
browser" path and the strongest guard for the #1 cluster. Two subtleties this
|
||||
surfaced, now handled by the mock: the cleared page must exceed the bypasser's
|
||||
`_LOADING_BODY_LENGTH_MAX` (50 chars of innerText) or it loops as "still loading",
|
||||
and the AA detail page must satisfy the brittle `original_nodes[-6]` parse (#880).
|
||||
- **DoH** on at boot.
|
||||
- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`,
|
||||
`/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a
|
||||
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real
|
||||
qBittorrent completes the download over HTTP (no tracker/peer/seeder), and
|
||||
shelfmark's completion detection + file move lands the book in `/books`. The
|
||||
webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked
|
||||
against shelfmark's own bencode/infohash parser in
|
||||
`tests/download/test_webseed_torrent_generator.py`), and the whole
|
||||
prowlarr→qBittorrent path is configured declaratively via env (`env/full.env`).
|
||||
|
||||
## Components
|
||||
|
||||
| Path | Purpose |
|
||||
|---|---|
|
||||
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. |
|
||||
| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. |
|
||||
| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. |
|
||||
| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. |
|
||||
| `mocks/fixtures/*.html` | AA search/detail HTML in the **exact** shape the parser expects, plus drift/empty/no-files variants. |
|
||||
| `docker-compose.e2e.yml` | The stack; optional services gated by compose profiles, static IPs for DNS determinism. |
|
||||
| `dns/Corefile*`, `dns/mock.test.db` | coredns zones — working + ISP-block (NXDOMAIN). |
|
||||
| `env/*.env` | The config profiles (matrix rows). |
|
||||
| `suite/` | The pytest harness + cluster tests. |
|
||||
| `run-e2e.sh` / `run-matrix.sh` | Boot one profile / loop the matrix. |
|
||||
| `build-images.sh` | Build the heavy image once (`make e2e-platform-build`); reused via `E2E_NO_BUILD=1`. |
|
||||
|
||||
### Fault injection
|
||||
The mock AA reproduces historical bugs deterministically. Injection rides inside
|
||||
the search query as `E2EINJECT:<name>` (the app builds the AA URL itself and only
|
||||
forwards the user query as `q=`). Names: `no_files`, `empty`, `layout_drift`,
|
||||
`500`. The harness embeds them via `PlatformClient.direct_search(..., inject=...)`.
|
||||
|
||||
## Gating PRs (block merge on e2e failure)
|
||||
|
||||
The `.github/workflows/e2e-platform.yml` workflow runs on every PR. On a PR that
|
||||
touches relevant code (`shelfmark/**`, `Dockerfile`, `entrypoint.sh`, `tor.sh`,
|
||||
`tests/e2e/platform/**`) it runs the fast PR subset **and** the heavy `full`
|
||||
profile (real Chrome solving Cloudflare + DoH + real qBittorrent), then a single
|
||||
**`e2e required`** job aggregates them: it fails if any e2e job failed, and passes
|
||||
(so it never hangs) when the e2e jobs are skipped on an unrelated PR.
|
||||
|
||||
The workflow producing a failing check is **not enough on its own** — GitHub only
|
||||
*blocks merge* on checks listed in branch protection. A repo **admin** must, once:
|
||||
|
||||
- **UI:** Settings → Branches → branch protection rule for `main` →
|
||||
*Require status checks to pass before merging* → add **`e2e required`**.
|
||||
- **or `gh` (admin token):**
|
||||
```bash
|
||||
gh api -X PUT repos/calibrain/shelfmark/branches/main/protection \
|
||||
-H "Accept: application/vnd.github+json" --input - <<'JSON'
|
||||
{ "required_status_checks": { "strict": true, "contexts": ["e2e required"] },
|
||||
"enforce_admins": true, "required_pull_request_reviews": null, "restrictions": null }
|
||||
JSON
|
||||
```
|
||||
|
||||
After that, any failure in the e2e platform tests (including the `full` profile)
|
||||
blocks the PR from merging. Requiring just the one `e2e required` context covers
|
||||
the whole dynamic matrix, so the list never needs updating as profiles change.
|
||||
|
||||
## Known limitations / follow-ups
|
||||
|
||||
- **rTorrent.** Not in the matrix: its rakshasa-libtorrent has **no GetRight/webseed
|
||||
support**, so the hermetic webseed torrent (which qBittorrent/Transmission/Deluge
|
||||
all complete) leaves rTorrent stuck at 0%. Supporting it needs a real tracker +
|
||||
seeder (peer download) — a follow-up that the webseed design intentionally avoids.
|
||||
- **Usenet clients (SABnzbd/NZBGet).** Not yet covered — completing a usenet download
|
||||
hermetically needs a mock NNTP server serving the yEnc-encoded payload plus an NZB,
|
||||
which is a separate (larger) build than the torrent webseed path.
|
||||
- **Audiobook (cluster 7) in-stack.** ABB hardcodes `https://`, so it's covered
|
||||
offline (`tests/audiobookbay/test_scraper_contract.py`); an in-stack
|
||||
`audiobookbay` role needs the same self-signed-HTTPS plumbing the `dns-doh` profile
|
||||
now uses for DoH.
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build every buildable image in the e2e stack once (the heavy `shelfmark` image
|
||||
# plus the mock-* role images), so run-matrix.sh / run-e2e.sh with E2E_NO_BUILD=1
|
||||
# can reuse them instead of rebuilding the xvfb/chromium layer per profile.
|
||||
set -euo pipefail
|
||||
|
||||
PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$PLATFORM_DIR"
|
||||
|
||||
# Activate every profile that owns a buildable service so they all get built.
|
||||
# (Download clients, coredns, proxies are pre-built images — nothing to build.)
|
||||
export COMPOSE_PROFILES="bypasser-external,full,dns-doh"
|
||||
echo "==> building shelfmark + mock images (one cold build of the chromium layer)"
|
||||
docker compose -f docker-compose.e2e.yml build
|
||||
echo "==> done. Reuse with: E2E_NO_BUILD=1 ./run-e2e.sh env/<profile>.env"
|
||||
@@ -0,0 +1,13 @@
|
||||
# coredns: authoritative for mock.test, forwards everything else.
|
||||
mock.test:53 {
|
||||
file /zones/mock.test.db
|
||||
log
|
||||
errors
|
||||
}
|
||||
|
||||
. :53 {
|
||||
forward . 1.1.1.1 8.8.8.8
|
||||
cache 30
|
||||
log
|
||||
errors
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# coredns: ISP-DNS-block simulation. Resolves nothing under mock.test
|
||||
# (returns NXDOMAIN), so an app that relies on *system* DNS for book sources
|
||||
# fails — while an app that uses its own custom DNS resolver still works.
|
||||
# Regression harness for #1028 (internal bypasser used system DNS in subprocess).
|
||||
mock.test:53 {
|
||||
template IN ANY mock.test {
|
||||
rcode NXDOMAIN
|
||||
}
|
||||
log
|
||||
errors
|
||||
}
|
||||
|
||||
. :53 {
|
||||
# Everything else still resolves, so only the book-source host is "blocked".
|
||||
forward . 1.1.1.1 8.8.8.8
|
||||
log
|
||||
errors
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
$ORIGIN mock.test.
|
||||
$TTL 60
|
||||
@ IN SOA ns.mock.test. admin.mock.test. (
|
||||
1 ; serial
|
||||
7200 ; refresh
|
||||
3600 ; retry
|
||||
1209600 ; expire
|
||||
3600 ) ; minimum
|
||||
@ IN NS ns.mock.test.
|
||||
ns IN A 172.30.0.20
|
||||
aa IN A 172.30.0.10
|
||||
cf IN A 172.30.0.11
|
||||
doh IN A 172.30.0.21
|
||||
@@ -0,0 +1,332 @@
|
||||
# Shelfmark e2e Docker testing platform
|
||||
# ---------------------------------------------------------------------------
|
||||
# A hermetic stack: the app under test plus *controllable* dependencies
|
||||
# (fake Anna's Archive, a Cloudflare gate, a mock FlareSolverr bypasser, a DoH
|
||||
# responder, a DNS server, HTTP + SOCKS5 proxies, and a Tor profile).
|
||||
#
|
||||
# Config profiles are env-driven: pick a profile with an env file under env/
|
||||
# (it sets COMPOSE_PROFILES + the SM_* config the app boots with). The runner
|
||||
# brings the stack up per profile, runs the matching cluster tests, tears down.
|
||||
#
|
||||
# ./run-e2e.sh env/baseline.env
|
||||
# ./run-e2e.sh env/bypasser-external.env
|
||||
# ./run-e2e.sh env/dns-blocked.env
|
||||
#
|
||||
# See README.md for the full cluster x profile matrix.
|
||||
|
||||
x-mock-build: &mock-build
|
||||
build:
|
||||
context: ./mocks
|
||||
dockerfile: Dockerfile
|
||||
|
||||
networks:
|
||||
e2e:
|
||||
driver: bridge
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.30.0.0/24
|
||||
|
||||
services:
|
||||
# ----- App under test --------------------------------------------------- #
|
||||
shelfmark:
|
||||
build:
|
||||
context: ../../..
|
||||
dockerfile: Dockerfile
|
||||
target: ${SM_BUILD_TARGET:-shelfmark}
|
||||
container_name: e2e-shelfmark
|
||||
cap_add:
|
||||
- NET_ADMIN # required by tor.sh iptables when USING_TOR=true
|
||||
- NET_RAW
|
||||
environment:
|
||||
TZ: UTC
|
||||
DEBUG: "true"
|
||||
ONBOARDING: "false" # skip wizard; ephemeral storage
|
||||
DISABLE_LOCAL_AUTH: "${SM_DISABLE_LOCAL_AUTH:-true}"
|
||||
PUID: "${SM_PUID:-1000}"
|
||||
PGID: "${SM_PGID:-1000}"
|
||||
# --- source / mirror config (cluster 2/3) ---
|
||||
DIRECT_DOWNLOAD_ENABLED: "${SM_DIRECT_DOWNLOAD_ENABLED:-true}"
|
||||
AA_ADDITIONAL_URLS: "${SM_AA_URL:-http://mock-aa}"
|
||||
# --- bypasser config (cluster 1) ---
|
||||
USE_CF_BYPASS: "${SM_USE_CF_BYPASS:-false}"
|
||||
# Selects the external (FlareSolverr) bypasser; when false the in-image
|
||||
# Chrome (internal) bypasser is used. NOT derived from EXT_BYPASSER_URL.
|
||||
USING_EXTERNAL_BYPASSER: "${SM_USING_EXTERNAL_BYPASSER:-false}"
|
||||
EXT_BYPASSER_URL: "${SM_EXT_BYPASSER_URL:-}"
|
||||
# --- prowlarr indexer + torrent client (cluster 5, `full` profile) ---
|
||||
PROWLARR_ENABLED: "${SM_PROWLARR_ENABLED:-false}"
|
||||
PROWLARR_URL: "${SM_PROWLARR_URL:-}"
|
||||
PROWLARR_API_KEY: "${SM_PROWLARR_API_KEY:-}"
|
||||
PROWLARR_TORRENT_CLIENT: "${SM_PROWLARR_TORRENT_CLIENT:-}"
|
||||
QBITTORRENT_URL: "${SM_QBITTORRENT_URL:-}"
|
||||
QBITTORRENT_USERNAME: "${SM_QBITTORRENT_USERNAME:-}"
|
||||
QBITTORRENT_PASSWORD: "${SM_QBITTORRENT_PASSWORD:-}"
|
||||
QBITTORRENT_CATEGORY: "${SM_QBITTORRENT_CATEGORY:-}"
|
||||
DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT: "${SM_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT:-60}"
|
||||
# transmission / deluge / rtorrent (client-* profiles)
|
||||
TRANSMISSION_URL: "${SM_TRANSMISSION_URL:-}"
|
||||
TRANSMISSION_USERNAME: "${SM_TRANSMISSION_USERNAME:-}"
|
||||
TRANSMISSION_PASSWORD: "${SM_TRANSMISSION_PASSWORD:-}"
|
||||
DELUGE_HOST: "${SM_DELUGE_HOST:-}"
|
||||
DELUGE_PORT: "${SM_DELUGE_PORT:-}"
|
||||
DELUGE_PASSWORD: "${SM_DELUGE_PASSWORD:-}"
|
||||
RTORRENT_URL: "${SM_RTORRENT_URL:-}"
|
||||
# --- DNS / DoH (config cluster) ---
|
||||
CUSTOM_DNS: "${SM_CUSTOM_DNS:-}"
|
||||
CUSTOM_DNS_MANUAL: "${SM_CUSTOM_DNS_MANUAL:-}"
|
||||
USE_DOH: "${SM_USE_DOH:-false}"
|
||||
# Disable TLS verification so the in-stack DoH-over-HTTPS mock (self-signed)
|
||||
# is accepted in the dns-doh profile. Default keeps verification ON.
|
||||
CERTIFICATE_VALIDATION: "${SM_CERTIFICATE_VALIDATION:-enabled}"
|
||||
# --- proxy (config cluster) ---
|
||||
PROXY_MODE: "${SM_PROXY_MODE:-none}"
|
||||
HTTP_PROXY_URL: "${SM_HTTP_PROXY:-}"
|
||||
HTTP_PROXY: "${SM_HTTP_PROXY:-}"
|
||||
SOCKS5_PROXY: "${SM_SOCKS5_PROXY:-}"
|
||||
NO_PROXY: "${SM_NO_PROXY:-}"
|
||||
# --- tor (cluster 1/6) ---
|
||||
USING_TOR: "${SM_USING_TOR:-false}"
|
||||
ports:
|
||||
- "8084:8084"
|
||||
volumes:
|
||||
- ./.state/config:/config
|
||||
- ./.state/books:/books
|
||||
- ${SM_DOWNLOADS_HOST_DIR:-./.state/downloads}:/downloads
|
||||
- ./.state/tmp:/tmp/shelfmark
|
||||
networks:
|
||||
- e2e
|
||||
dns:
|
||||
# When a DNS profile is active, point the container's system resolver at
|
||||
# our controllable server; otherwise Docker's embedded DNS (127.0.0.11).
|
||||
- ${SM_SYSTEM_DNS:-127.0.0.11}
|
||||
extra_hosts:
|
||||
# Redirect the DoH provider hostname to the in-stack mock-doh (dns-doh
|
||||
# profile). Harmless elsewhere — only the dns-doh profile enables DoH against
|
||||
# the cloudflare provider, and /etc/hosts is consulted before the resolver.
|
||||
- "cloudflare-dns.com:172.30.0.21"
|
||||
restart: "no"
|
||||
|
||||
# ----- Fake Anna's Archive origin (always on) --------------------------- #
|
||||
mock-aa:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-aa
|
||||
environment:
|
||||
MOCK_ROLE: origin-aa
|
||||
# When set (the `full` profile sets it to the CF gate), AA slow-download
|
||||
# links point through Cloudflare so a real download forces the internal
|
||||
# Chrome bypasser to solve the challenge. Empty -> same-origin (no CF).
|
||||
SLOW_DOWNLOAD_BASE: "${SM_SLOW_DOWNLOAD_BASE:-}"
|
||||
AA_FILE_BASE: "http://mock-aa"
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.10
|
||||
aliases:
|
||||
- aa.mock.test
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://localhost/healthz')"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
# ----- Cloudflare gate (profiles: bypasser-external, full) -------------- #
|
||||
# `full` puts AA's slow-download behind this gate so the real Chrome bypasser
|
||||
# must solve it; `bypasser-external` puts AA search behind it for the
|
||||
# FlareSolverr negative/positive controls.
|
||||
mock-cf:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-cf
|
||||
profiles: ["bypasser-external", "full"]
|
||||
environment:
|
||||
MOCK_ROLE: cloudflare
|
||||
ORIGIN_INTERNAL_URL: http://mock-aa
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.11
|
||||
aliases:
|
||||
- cf.mock.test
|
||||
|
||||
# ----- Mock FlareSolverr external bypasser (profile: bypasser-external) -- #
|
||||
mock-flaresolverr:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-flaresolverr
|
||||
profiles: ["bypasser-external"]
|
||||
environment:
|
||||
MOCK_ROLE: flaresolverr
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.12
|
||||
|
||||
# NOTE: the mock `doh` role lives in mock_services.py and is exercised by
|
||||
# tests/download/test_doh_resolver_mock.py (real DoHResolver over localhost HTTP).
|
||||
# An in-stack DoH service is intentionally absent — see README "Known limitations".
|
||||
|
||||
# ----- DNS server (profile: dns-manual) --------------------------------- #
|
||||
coredns:
|
||||
image: coredns/coredns:1.11.1
|
||||
container_name: e2e-coredns
|
||||
profiles: ["dns-manual"]
|
||||
command: ["-conf", "/Corefile"]
|
||||
volumes:
|
||||
- ./dns/Corefile:/Corefile:ro
|
||||
- ./dns/mock.test.db:/zones/mock.test.db:ro
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.20
|
||||
|
||||
# ----- DNS server that NXDOMAINs the AA host (profile: dns-blocked) ------ #
|
||||
# Simulates ISP DNS blocking (#1028); the app must fall back (DoH / direct).
|
||||
coredns-blocked:
|
||||
image: coredns/coredns:1.11.1
|
||||
container_name: e2e-coredns-blocked
|
||||
profiles: ["dns-blocked"]
|
||||
command: ["-conf", "/Corefile"]
|
||||
volumes:
|
||||
- ./dns/Corefile.blocked:/Corefile:ro
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.22
|
||||
|
||||
# ----- HTTP proxy (profile: proxy-http) --------------------------------- #
|
||||
tinyproxy:
|
||||
image: monokal/tinyproxy:latest
|
||||
container_name: e2e-tinyproxy
|
||||
profiles: ["proxy-http"]
|
||||
command: ANY
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.30
|
||||
|
||||
# ----- SOCKS5 proxy (profile: proxy-socks) ------------------------------ #
|
||||
microsocks:
|
||||
image: vimagick/microsocks:latest
|
||||
container_name: e2e-microsocks
|
||||
profiles: ["proxy-socks"]
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.31
|
||||
|
||||
# ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- #
|
||||
# Minimal Prowlarr API returning one torrent release whose .torrent is a
|
||||
# webseed pointing at mock-aa. Drives the real torrent-client download in the
|
||||
# `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles.
|
||||
mock-prowlarr:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-prowlarr
|
||||
profiles: ["full", "client-transmission", "client-deluge", "client-qbittorrent-delayed"]
|
||||
environment:
|
||||
MOCK_ROLE: prowlarr
|
||||
AA_INTERNAL_URL: http://mock-aa
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.40
|
||||
aliases:
|
||||
- prowlarr.mock.test
|
||||
|
||||
# ----- Real qBittorrent download client (profiles: full, delayed path) --- #
|
||||
# Auth is bypassed for the e2e subnet (qBittorrent.conf) so shelfmark connects
|
||||
# without juggling the image's random temp password.
|
||||
qbittorrent:
|
||||
image: lscr.io/linuxserver/qbittorrent:latest
|
||||
container_name: e2e-qbittorrent
|
||||
profiles: ["full", "client-qbittorrent-delayed"]
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: UTC
|
||||
WEBUI_PORT: "8080"
|
||||
volumes:
|
||||
- ./qbittorrent/qBittorrent.conf:/config/qBittorrent/qBittorrent.conf
|
||||
# Normally shared with Shelfmark. In the delayed profile this points at a
|
||||
# client-only directory and the sync sidecar makes it visible later.
|
||||
- ${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/downloads}:/downloads
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# ----- Delayed file visibility sidecar (profile: client-qbittorrent-delayed) #
|
||||
# Simulates a seedbox/remote-sync workflow: qBittorrent writes into one host
|
||||
# directory while Shelfmark sees a separate /downloads directory that is synced
|
||||
# only after a delay. This exercises DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT.
|
||||
delayed-download-sync:
|
||||
image: alpine:3.20
|
||||
container_name: e2e-delayed-download-sync
|
||||
profiles: ["client-qbittorrent-delayed"]
|
||||
environment:
|
||||
SYNC_DELAY_SECONDS: "${SM_DELAYED_DOWNLOAD_SYNC_SECONDS:-20}"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
mkdir -p /synced
|
||||
while true; do
|
||||
found=0
|
||||
for path in /remote/*; do
|
||||
[ -e "$${path}" ] || continue
|
||||
found=1
|
||||
before="$$(du -sk /remote 2>/dev/null | awk '{print $$1}')"
|
||||
sleep "$${SYNC_DELAY_SECONDS}"
|
||||
after="$$(du -sk /remote 2>/dev/null | awk '{print $$1}')"
|
||||
if [ -n "$${before}" ] && [ "$${before}" = "$${after}" ]; then
|
||||
cp -a /remote/. /synced/ 2>/dev/null || true
|
||||
fi
|
||||
break
|
||||
done
|
||||
[ "$${found}" = "1" ] || sleep 1
|
||||
done
|
||||
volumes:
|
||||
- ${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/downloads}:/remote:ro
|
||||
- ${SM_DOWNLOADS_HOST_DIR:-./.state/downloads}:/synced
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# ----- Real Transmission client (profile: client-transmission) ---------- #
|
||||
# Same webseed torrent + mock Prowlarr as `full`, different real client.
|
||||
transmission:
|
||||
image: lscr.io/linuxserver/transmission:latest
|
||||
container_name: e2e-transmission
|
||||
profiles: ["client-transmission"]
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: UTC
|
||||
USER: admin
|
||||
PASS: admin
|
||||
volumes:
|
||||
- ./.state/downloads:/downloads
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# ----- Real Deluge client (profile: client-deluge) ---------------------- #
|
||||
# shelfmark talks to deluge-web (default WebUI password "deluge"), which
|
||||
# connects to the bundled daemon — no daemon auth-file juggling.
|
||||
deluge:
|
||||
image: lscr.io/linuxserver/deluge:latest
|
||||
container_name: e2e-deluge
|
||||
profiles: ["client-deluge"]
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: UTC
|
||||
volumes:
|
||||
- ./.state/downloads:/downloads
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# NOTE: rTorrent is intentionally absent — its rakshasa-libtorrent has no
|
||||
# GetRight/webseed support, so the hermetic webseed torrent can't drive it.
|
||||
# Supporting it would need a real tracker + seeder (see README limitations).
|
||||
|
||||
# ----- DoH-over-HTTPS responder (profile: dns-doh) ---------------------- #
|
||||
# In-stack DoH: serves the DNS JSON API over HTTPS with a self-signed cert.
|
||||
# shelfmark reaches it because the profile (a) maps the DoH provider hostname
|
||||
# to this container via extra_hosts and (b) sets CERTIFICATE_VALIDATION=disabled.
|
||||
mock-doh:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-doh
|
||||
profiles: ["dns-doh"]
|
||||
environment:
|
||||
MOCK_ROLE: doh
|
||||
DOH_TLS: "1"
|
||||
DOH_MAP: "aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11"
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.21
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
# Profile: baseline
|
||||
# Direct connection to the fake Anna's Archive, no bypasser, no proxy, system DNS.
|
||||
# Exercises clusters 2/3 (search + AA parsing) and the download happy path.
|
||||
COMPOSE_PROFILES=
|
||||
SM_AA_URL=http://mock-aa
|
||||
SM_USE_CF_BYPASS=false
|
||||
SM_PROXY_MODE=none
|
||||
E2E_PROFILE=baseline
|
||||
@@ -0,0 +1,9 @@
|
||||
# Profile: bypasser-disabled (cluster 1 negative control)
|
||||
# AA behind the Cloudflare gate but the bypasser is OFF. Proves the gate really
|
||||
# challenges: search must FAIL clearly here, which is what makes the
|
||||
# bypasser-external success meaningful (and guards #202/#410 "uses AA when CF
|
||||
# bypass disabled" / "config not adhered").
|
||||
COMPOSE_PROFILES=bypasser-external
|
||||
SM_AA_URL=http://cf.mock.test
|
||||
SM_USE_CF_BYPASS=false
|
||||
E2E_PROFILE=bypasser-disabled
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Profile: bypasser-external (cluster 1)
|
||||
# AA is served behind a Cloudflare gate. The app must detect the challenge and
|
||||
# route through the external bypasser (mock FlareSolverr), which returns solved
|
||||
# HTML + cf_clearance. Regression surface: #284 #226 #202 #1030 #410 #369 #267.
|
||||
COMPOSE_PROFILES=bypasser-external
|
||||
SM_AA_URL=http://cf.mock.test
|
||||
SM_USE_CF_BYPASS=true
|
||||
# Must be set explicitly — shelfmark does NOT derive it from EXT_BYPASSER_URL.
|
||||
# Without it the app falls back to the in-image Chrome bypasser and never calls
|
||||
# FlareSolverr, so this profile would silently exercise the wrong path.
|
||||
SM_USING_EXTERNAL_BYPASSER=true
|
||||
SM_EXT_BYPASSER_URL=http://mock-flaresolverr
|
||||
E2E_PROFILE=bypasser-external
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Profile: client-deluge (cluster 5 — real Deluge client)
|
||||
# shelfmark talks to deluge-web (default WebUI password "deluge"), which connects
|
||||
# to the bundled daemon — no daemon auth-file juggling.
|
||||
COMPOSE_PROFILES=client-deluge
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=deluge
|
||||
SM_DELUGE_HOST=deluge
|
||||
SM_DELUGE_PORT=8112
|
||||
SM_DELUGE_PASSWORD=deluge
|
||||
E2E_DOWNLOAD_TIMEOUT=180
|
||||
E2E_PROFILE=client-deluge
|
||||
@@ -0,0 +1,21 @@
|
||||
# Profile: client-qbittorrent-delayed (cluster 5 — delayed completed-path visibility)
|
||||
# Same mock Prowlarr + webseed torrent as `full`, but qBittorrent writes to a
|
||||
# directory Shelfmark cannot see until the delayed sync sidecar copies it over.
|
||||
# This models seedbox / remote-sync setups where the client reports completion
|
||||
# before the completed path exists inside Shelfmark.
|
||||
COMPOSE_PROFILES=client-qbittorrent-delayed
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=qbittorrent
|
||||
SM_QBITTORRENT_URL=http://qbittorrent:8080
|
||||
SM_QBITTORRENT_USERNAME=admin
|
||||
SM_QBITTORRENT_PASSWORD=adminadmin
|
||||
SM_QBITTORRENT_CATEGORY=books
|
||||
|
||||
# The sync delay must be longer than the default client path polling interval
|
||||
# so Shelfmark actually enters the completed-path wait loop.
|
||||
SM_DELAYED_DOWNLOAD_SYNC_SECONDS=20
|
||||
SM_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT=90
|
||||
E2E_DOWNLOAD_TIMEOUT=180
|
||||
E2E_PROFILE=client-qbittorrent-delayed
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Profile: client-transmission (cluster 5 — real Transmission client)
|
||||
# Same mock Prowlarr + webseed torrent as `full`, but the torrent goes to a real
|
||||
# Transmission. Lean (no Chrome/CF/DoH), so it boots fast.
|
||||
COMPOSE_PROFILES=client-transmission
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=transmission
|
||||
SM_TRANSMISSION_URL=http://transmission:9091
|
||||
SM_TRANSMISSION_USERNAME=admin
|
||||
SM_TRANSMISSION_PASSWORD=admin
|
||||
E2E_DOWNLOAD_TIMEOUT=180
|
||||
E2E_PROFILE=client-transmission
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
# Profile: dns-blocked (config cluster: ISP DNS block — #1028 regression)
|
||||
# The container's *system* resolver (coredns-blocked) NXDOMAINs the AA host,
|
||||
# but the app's *custom* DNS resolver (coredns) resolves it fine. A correct app
|
||||
# reaches AA via custom DNS; a regressed one (system DNS in subprocess) fails.
|
||||
COMPOSE_PROFILES=dns-manual,dns-blocked
|
||||
SM_AA_URL=http://aa.mock.test
|
||||
SM_SYSTEM_DNS=172.30.0.22
|
||||
SM_CUSTOM_DNS=manual
|
||||
SM_CUSTOM_DNS_MANUAL=172.30.0.20
|
||||
E2E_PROFILE=dns-blocked
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
# Profile: dns-doh (config cluster — in-stack DNS-over-HTTPS, end to end)
|
||||
#
|
||||
# Proves the app's DoH path resolves a name over real HTTPS:
|
||||
# * system resolver = coredns-blocked, which NXDOMAINs aa.mock.test, so the host
|
||||
# can ONLY be resolved via DoH;
|
||||
# * CUSTOM_DNS=cloudflare + USE_DOH=true => the app queries the cloudflare DoH
|
||||
# endpoint, which docker-compose `extra_hosts` redirects to the in-stack
|
||||
# mock-doh (HTTPS, self-signed). CERTIFICATE_VALIDATION=disabled accepts it.
|
||||
# * mock-doh maps aa.mock.test -> mock-aa, so the search reaches AA — only
|
||||
# possible if DoH resolution worked.
|
||||
COMPOSE_PROFILES=dns-doh,dns-blocked
|
||||
SM_AA_URL=http://aa.mock.test
|
||||
SM_SYSTEM_DNS=172.30.0.22
|
||||
SM_CUSTOM_DNS=cloudflare
|
||||
SM_USE_DOH=true
|
||||
SM_CERTIFICATE_VALIDATION=disabled
|
||||
E2E_PROFILE=dns-doh
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
# Profile: dns-manual (config cluster: custom DNS)
|
||||
# AA host only resolves via the controllable coredns server; the app is told to
|
||||
# use it as a manual DNS provider. Verifies custom-DNS resolution (#108-style).
|
||||
COMPOSE_PROFILES=dns-manual
|
||||
SM_AA_URL=http://aa.mock.test
|
||||
SM_CUSTOM_DNS=manual
|
||||
SM_CUSTOM_DNS_MANUAL=172.30.0.20
|
||||
SM_SYSTEM_DNS=172.30.0.20
|
||||
E2E_PROFILE=dns-manual
|
||||
Vendored
+42
@@ -0,0 +1,42 @@
|
||||
# Profile: full (the "everything real" heavy profile — nightly/manual only)
|
||||
#
|
||||
# Test book: Moby-Dick (Herman Melville, public domain).
|
||||
#
|
||||
# What it exercises end to end, with NO mock bypasser:
|
||||
# 1. REAL Chrome internal bypasser: AA search/detail are reachable (mock-aa), but
|
||||
# the AA *slow-download* links point through the Cloudflare gate. So a real
|
||||
# DOWNLOAD forces the in-image headless Chrome (seleniumbase CDP) to solve the
|
||||
# JS challenge and harvest cf_clearance — the literal "spin a chrome browser"
|
||||
# path. Success = Moby-Dick lands in /books, which is only possible if Chrome
|
||||
# solved the gate.
|
||||
# 2. DoH enabled (USE_DOH) so the DNS-over-HTTPS code path runs at boot.
|
||||
# 3. REAL download client: a mock Prowlarr returns a webseed .torrent; a real
|
||||
# qBittorrent downloads the payload over HTTP from mock-aa and shelfmark
|
||||
# moves the completed file into /books.
|
||||
#
|
||||
# Heavy (Chrome + qBittorrent), so it is excluded from the PR matrix.
|
||||
COMPOSE_PROFILES=full
|
||||
|
||||
# AA search/detail reachable directly; the DOWNLOAD goes through the CF gate so the
|
||||
# internal Chrome bypasser must solve it (search uses no bypasser, by design).
|
||||
SM_AA_URL=http://mock-aa
|
||||
SM_SLOW_DOWNLOAD_BASE=http://cf.mock.test
|
||||
SM_USE_CF_BYPASS=true
|
||||
SM_USING_EXTERNAL_BYPASSER=false
|
||||
|
||||
# DNS over HTTPS on.
|
||||
SM_USE_DOH=true
|
||||
|
||||
# Prowlarr indexer + real qBittorrent client.
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=qbittorrent
|
||||
SM_QBITTORRENT_URL=http://qbittorrent:8080
|
||||
SM_QBITTORRENT_USERNAME=admin
|
||||
SM_QBITTORRENT_PASSWORD=adminadmin
|
||||
SM_QBITTORRENT_CATEGORY=books
|
||||
|
||||
# The real client download needs more headroom than a direct HTTP fetch.
|
||||
E2E_DOWNLOAD_TIMEOUT=300
|
||||
E2E_PROFILE=full
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
# Profile: proxy-http (config cluster: HTTP proxy — #956 proxy-ignored regressions)
|
||||
# All app egress routes through tinyproxy. Search/download must still succeed,
|
||||
# and traffic must actually traverse the proxy.
|
||||
COMPOSE_PROFILES=proxy-http
|
||||
SM_AA_URL=http://mock-aa
|
||||
SM_PROXY_MODE=http
|
||||
SM_HTTP_PROXY=http://tinyproxy:8888
|
||||
E2E_PROFILE=proxy-http
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
# Profile: proxy-socks (config cluster: SOCKS5 proxy)
|
||||
# All app egress routes through a SOCKS5 proxy (microsocks).
|
||||
COMPOSE_PROFILES=proxy-socks
|
||||
SM_AA_URL=http://mock-aa
|
||||
SM_PROXY_MODE=socks5
|
||||
SM_SOCKS5_PROXY=socks5://microsocks:1080
|
||||
E2E_PROFILE=proxy-socks
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
# Profile: tor (cluster 1/6: Tor boot — #1021 #940 #801 regressions)
|
||||
# Boots the app with transparent Tor routing. The platform asserts the container
|
||||
# reaches a healthy /api/health WITHOUT boot-looping or pegging CPU, and that the
|
||||
# app reports DNS/proxy as Tor-managed. (Real Tor egress is slow/flaky in CI, so
|
||||
# this profile is startup-correctness focused; see tor-full for real egress.)
|
||||
COMPOSE_PROFILES=
|
||||
SM_USING_TOR=true
|
||||
SM_AA_URL=http://mock-aa
|
||||
E2E_PROFILE=tor
|
||||
@@ -0,0 +1,25 @@
|
||||
# Mock services image for the Shelfmark e2e platform.
|
||||
# One image, many roles (selected via MOCK_ROLE): origin-aa, cloudflare,
|
||||
# flaresolverr, prowlarr, doh. See mock_services.py for the role contracts.
|
||||
FROM python:3.14-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY mock_services.py .
|
||||
# Needed by the origin-aa role to build the `full` profile's webseed .torrent.
|
||||
COPY make_webseed_torrent.py .
|
||||
COPY fixtures ./fixtures
|
||||
|
||||
ENV PORT=80
|
||||
ENV MOCK_ROLE=all
|
||||
|
||||
# gunicorn keeps the mock responsive under concurrent e2e load.
|
||||
RUN pip install --no-cache-dir gunicorn
|
||||
EXPOSE 80
|
||||
EXPOSE 443
|
||||
# DOH_TLS=1 (the `doh` role) self-signs a cert and serves HTTPS on 443; every
|
||||
# other role serves plain HTTP on $PORT via gunicorn.
|
||||
CMD ["sh", "-c", "if [ \"$DOH_TLS\" = 1 ]; then python mock_services.py; else gunicorn -w 2 -b 0.0.0.0:${PORT} mock_services:app; fi"]
|
||||
@@ -0,0 +1,15 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><title>Anna's Archive (mock) - __MD5__</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<div>
|
||||
<div><img src="/img/cover-__MD5__.jpg"></div>
|
||||
<div class="main-inner">
|
||||
<div><div class="md-meta"><div><div><span>Language</span><span>English [en]</span></div><div><span>Year</span><span>1851</span></div></div></div><div>🔍 Moby Dick</div><div>epub · 1.2 MB · fiction</div><div><a href="__SLOW_BASE__/slow_download/__MD5__/0/0">Slow Partner Server #1</a> — no waitlist, but the download is slow</div><div><a href="__SLOW_BASE__/slow_download/__MD5__/0/1">Slow Partner Server #2</a> — waitlist, but faster</div><div><a href="/get.php?md5=__MD5__&key=e2ekey">GET</a></div></div>
|
||||
</div>
|
||||
<div class="js-md5-top-box-description">Moby-Dick; or, The Whale by Herman Melville.</div>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,9 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><title>Anna's Archive (mock)</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<div>No files found.</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,27 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><title>Anna's Archive (mock - layout drift)</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<!--
|
||||
Simulates Anna's Archive changing its DOM (the recurring root cause behind
|
||||
#878/#879/#880 and the repeated "Fix AA ... after they changed layout" PRs).
|
||||
The result rows now use <div class="result"> cards instead of a <table> of
|
||||
<tr>/<td>. A parser with hardcoded table/cell indices yields ZERO rows here.
|
||||
The matching e2e test asserts the app FAILS LOUDLY (SearchUnavailableError /
|
||||
explicit "no results" surfaced to the client) rather than silently returning
|
||||
an empty list that users read as "book doesn't exist".
|
||||
-->
|
||||
<div class="results">
|
||||
<div class="result">
|
||||
<a href="/md5/cccccccccccccccccccccccccccccccc">
|
||||
<h3>Drifted Title</h3>
|
||||
<span class="author">Brandon Sanderson</span>
|
||||
<span class="ext">epub</span>
|
||||
<span class="size">1.2 MB</span>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,41 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><title>Anna's Archive (mock)</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<!-- NO_FILES_MARKER -->
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a href="/md5/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"><img src="/img/cover1.jpg"></a></td>
|
||||
<td><span>__QUERY__</span></td>
|
||||
<td><span>Herman Melville</span></td>
|
||||
<td><span>Harper & Brothers</span></td>
|
||||
<td><span>2024</span></td>
|
||||
<td><span>-</span></td>
|
||||
<td><span>-</span></td>
|
||||
<td><span>English [en]</span></td>
|
||||
<td><span>fiction</span></td>
|
||||
<td><span>epub</span></td>
|
||||
<td><span>1.2 MB</span></td>
|
||||
<td><span>lgli/N:\fiction\en\E2E\__QUERY__.epub</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="/md5/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"><img src="/img/cover2.jpg"></a></td>
|
||||
<td><span>__QUERY__ (Annotated Edition)</span></td>
|
||||
<td><span>Herman Melville</span></td>
|
||||
<td><span>Harper & Brothers</span></td>
|
||||
<td><span>2023</span></td>
|
||||
<td><span>-</span></td>
|
||||
<td><span>-</span></td>
|
||||
<td><span>English [en]</span></td>
|
||||
<td><span>fiction</span></td>
|
||||
<td><span>pdf</span></td>
|
||||
<td><span>8.4 MB</span></td>
|
||||
<td><span>lgli/N:\fiction\en\E2E\__QUERY__.pdf</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,39 @@
|
||||
<!doctype html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<title>Just a moment...</title>
|
||||
<!-- No-JS fallback: a browser with scripting disabled just keeps waiting,
|
||||
exactly like a real managed challenge. The internal bypasser drives a real
|
||||
Chrome, so the script branch below is what actually clears it. -->
|
||||
<meta http-equiv="refresh" content="8">
|
||||
</head>
|
||||
<body>
|
||||
<div class="main-wrapper" role="main">
|
||||
<div class="main-content">
|
||||
<noscript>
|
||||
<div id="challenge-error-title">Enable JavaScript and cookies to continue</div>
|
||||
</noscript>
|
||||
<div id="cf-please-wait">
|
||||
<p>Checking if the site connection is secure</p>
|
||||
<p>Verifying you are human. This may take a few seconds.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
// Mimics a managed Cloudflare challenge: after a short "verification"
|
||||
// delay a real browser is issued cf_clearance, then the page reloads and
|
||||
// the (now cookie-bearing) request is passed through to the real origin.
|
||||
// A regression that ignores the challenge / bypasser never reaches here.
|
||||
window._cf_chl_opt = { cType: 'managed' };
|
||||
(function () {
|
||||
function solve() {
|
||||
document.cookie = 'cf_clearance=e2e-cleared-token; path=/; SameSite=Lax';
|
||||
// Reload so the cleared cookie is sent on the next request.
|
||||
window.location.reload();
|
||||
}
|
||||
// Small delay so the bypasser observes a genuine challenge first.
|
||||
setTimeout(solve, 1200);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Generate a single-file BitTorrent metainfo (.torrent) with a BEP-19 webseed.
|
||||
|
||||
Used by the e2e platform's ``full`` profile so a *real* torrent client
|
||||
(qBittorrent) can complete a *real* download hermetically: the torrent carries no
|
||||
tracker and a single ``url-list`` webseed pointing at the mock origin's HTTP file
|
||||
endpoint, so libtorrent fetches the payload over HTTP — no tracker, peer, or
|
||||
seeder container required.
|
||||
|
||||
Stdlib-only (the mock image does not install shelfmark). The unit test cross-checks
|
||||
this encoder against shelfmark's own ``bencode_decode`` /
|
||||
``extract_info_hash_from_torrent`` so a divergence in either is caught.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
DEFAULT_PIECE_LENGTH = 16384 # 16 KiB — fine for the tiny e2e payload
|
||||
|
||||
|
||||
def bencode(value: object) -> bytes:
|
||||
"""Minimal bencode encoder (int / bytes / str / list / dict)."""
|
||||
if isinstance(value, bool): # guard: bool is an int subclass
|
||||
raise TypeError("bool is not bencodable")
|
||||
if isinstance(value, int):
|
||||
return b"i" + str(value).encode() + b"e"
|
||||
if isinstance(value, bytes):
|
||||
return str(len(value)).encode() + b":" + value
|
||||
if isinstance(value, str):
|
||||
return bencode(value.encode("utf-8"))
|
||||
if isinstance(value, list):
|
||||
return b"l" + b"".join(bencode(item) for item in value) + b"e"
|
||||
if isinstance(value, dict):
|
||||
out = b"d"
|
||||
for key in sorted(value): # bencode dict keys must be sorted
|
||||
key_bytes = key.encode("utf-8") if isinstance(key, str) else key
|
||||
out += bencode(key_bytes) + bencode(value[key])
|
||||
return out + b"e"
|
||||
raise TypeError(f"Cannot bencode value of type {type(value).__name__}")
|
||||
|
||||
|
||||
def _pieces(data: bytes, piece_length: int) -> bytes:
|
||||
return b"".join(
|
||||
hashlib.sha1(data[i : i + piece_length]).digest() for i in range(0, len(data), piece_length)
|
||||
)
|
||||
|
||||
|
||||
def build_info_dict(name: str, data: bytes, piece_length: int = DEFAULT_PIECE_LENGTH) -> dict:
|
||||
return {
|
||||
"name": name,
|
||||
"piece length": piece_length,
|
||||
"length": len(data),
|
||||
"pieces": _pieces(data, piece_length),
|
||||
}
|
||||
|
||||
|
||||
def build_webseed_torrent(
|
||||
name: str,
|
||||
data: bytes,
|
||||
webseed_url: str,
|
||||
*,
|
||||
piece_length: int = DEFAULT_PIECE_LENGTH,
|
||||
) -> bytes:
|
||||
"""Build a tracker-less single-file .torrent whose only source is a webseed.
|
||||
|
||||
Args:
|
||||
name: file name inside the torrent (e.g. ``sample-book.epub``).
|
||||
data: the exact file bytes the webseed URL must serve.
|
||||
webseed_url: BEP-19 url-list entry — the direct HTTP URL for ``data``.
|
||||
"""
|
||||
info = build_info_dict(name, data, piece_length)
|
||||
metainfo = {
|
||||
"info": info,
|
||||
# Single-entry webseed. For a single-file torrent the url-list entry is the
|
||||
# direct file URL, so it must serve exactly ``data``.
|
||||
"url-list": [webseed_url],
|
||||
"comment": "shelfmark e2e webseed torrent",
|
||||
"created by": "shelfmark-e2e",
|
||||
}
|
||||
return bencode(metainfo)
|
||||
|
||||
|
||||
def info_hash(torrent_bytes: bytes) -> str:
|
||||
"""The btih (SHA1 of the bencoded ``info`` dict) as a hex string.
|
||||
|
||||
Re-encodes via a tiny scan so we don't need a full decoder here.
|
||||
"""
|
||||
marker = b"4:infod"
|
||||
start = torrent_bytes.find(marker)
|
||||
if start < 0:
|
||||
raise ValueError("no info dict found in torrent")
|
||||
info_start = start + len(b"4:info")
|
||||
# The info value begins at 'd'; find its matching 'e' by bencode-aware scan.
|
||||
end = _scan_bencoded(torrent_bytes, info_start)
|
||||
return hashlib.sha1(torrent_bytes[info_start:end]).hexdigest()
|
||||
|
||||
|
||||
def _scan_bencoded(buf: bytes, pos: int) -> int:
|
||||
"""Return the index just past the bencoded value starting at ``pos``."""
|
||||
token = buf[pos : pos + 1]
|
||||
if token == b"i":
|
||||
return buf.index(b"e", pos) + 1
|
||||
if token in (b"l", b"d"):
|
||||
pos += 1
|
||||
while buf[pos : pos + 1] != b"e":
|
||||
if token == b"d": # dicts: key then value
|
||||
pos = _scan_bencoded(buf, pos)
|
||||
pos = _scan_bencoded(buf, pos)
|
||||
return pos + 1
|
||||
# byte string: <len>:<bytes>
|
||||
colon = buf.index(b":", pos)
|
||||
length = int(buf[pos:colon])
|
||||
return colon + 1 + length
|
||||
@@ -0,0 +1,577 @@
|
||||
"""Controllable mock services for the Shelfmark e2e Docker platform.
|
||||
|
||||
A single Flask app that plays one of several *roles*, selected by the
|
||||
``MOCK_ROLE`` environment variable. Running one image with different roles keeps
|
||||
the platform image small and the behaviour in one auditable place.
|
||||
|
||||
Roles
|
||||
-----
|
||||
``origin-aa`` Fake Anna's Archive: search results table, ``/md5/<id>`` detail
|
||||
pages, and a downloadable book file. The HTML mirrors the real
|
||||
selectors the parser depends on (``<tr>`` rows, last-cell
|
||||
distant path, ``get.php?md5=..&key=..`` GET links) so parser
|
||||
drift (#878/#879/#880) is caught here. Supports fault injection
|
||||
via query flags to reproduce historical bugs deterministically.
|
||||
``cloudflare`` Cloudflare-protected origin: returns a 403 "Just a moment..."
|
||||
challenge page (with ``cf-mitigated: challenge``) until the
|
||||
request carries a ``cf_clearance`` cookie, then serves the real
|
||||
content. Exercises the app's CF detection + bypasser routing
|
||||
(#284, #226, #202, #1030) without running real CF JS.
|
||||
``flaresolverr`` Mock FlareSolverr implementing the ``/v1`` contract. It fetches
|
||||
the requested URL *with* a clearance cookie and returns the
|
||||
solved HTML + cookies, so ``_fetch_via_bypasser`` runs end to
|
||||
end deterministically (no headless Chrome needed in CI).
|
||||
``doh`` Minimal DNS-over-HTTPS (RFC 8484 + Google JSON) responder used
|
||||
to verify the USE_DOH path resolves mock domains even when the
|
||||
system resolver is poisoned (#1028).
|
||||
``all`` Mounts every role at once (default; handy for local poking).
|
||||
|
||||
Fault injection (origin-aa) rides INSIDE the search query as an
|
||||
``E2EINJECT:<name>`` token, because the app builds the AA URL itself and only
|
||||
forwards the user query as ``q=``. The harness embeds it (see
|
||||
PlatformClient.direct_search); the mock strips it before rendering:
|
||||
``no_files`` -> renders the literal "No files found." alongside a real row
|
||||
(regression for the false-positive check).
|
||||
``empty`` -> renders an empty results page (true "No files found").
|
||||
``layout_drift`` -> renders a structurally-changed page (cards, no <table>) so
|
||||
a hardcoded-index parser yields zero rows — the app must
|
||||
fail loudly, not silently (#878/#879/#880).
|
||||
``500`` -> returns HTTP 500 (mirror failover path).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
from flask import Flask, Response, jsonify, make_response, request
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
ROLE = os.environ.get("MOCK_ROLE", "all").strip().lower()
|
||||
# Hostname the flaresolverr/cloudflare roles use to reach the AA origin from
|
||||
# inside the compose network.
|
||||
ORIGIN_INTERNAL_URL = os.environ.get("ORIGIN_INTERNAL_URL", "http://mock-aa")
|
||||
CLEARANCE_COOKIE = "cf_clearance"
|
||||
CLEARANCE_VALUE = "e2e-cleared-token"
|
||||
|
||||
# Test book: *Moby-Dick* by Herman Melville (public domain), used both as the
|
||||
# webseed-torrent payload (qBittorrent path) and the AA slow-download payload (real
|
||||
# Chrome / Cloudflare path). PAYLOAD_NAME/URL must stay in sync between what mock-aa
|
||||
# serves and what the torrent's url-list references.
|
||||
PAYLOAD_NAME = "moby-dick.epub"
|
||||
PAYLOAD_URL = f"{os.environ.get('PAYLOAD_PUBLIC_URL', 'http://mock-aa')}/payload/{PAYLOAD_NAME}"
|
||||
|
||||
# Base for AA slow-download links on the detail page. When set to the Cloudflare
|
||||
# gate (the `full` profile sets it to http://cf.mock.test), the *download* — not the
|
||||
# search — is forced through the gate, so a real download triggers the internal
|
||||
# Chrome bypasser to solve the challenge. Empty -> same-origin (no CF).
|
||||
SLOW_DOWNLOAD_BASE = os.environ.get("SLOW_DOWNLOAD_BASE", "").rstrip("/")
|
||||
|
||||
# Real public-domain opening of Moby-Dick (Chapter 1, "Loomings").
|
||||
_MOBY_DICK_TEXT = (
|
||||
"Call me Ishmael. Some years ago—never mind how long precisely—having little "
|
||||
"or no money in my purse, and nothing particular to interest me on shore, I "
|
||||
"thought I would sail about a little and see the watery part of the world. It "
|
||||
"is a way I have of driving off the spleen and regulating the circulation. "
|
||||
"Whenever I find myself growing grim about the mouth; whenever it is a damp, "
|
||||
"drizzly November in my soul; whenever I find myself involuntarily pausing "
|
||||
"before coffin warehouses, and bringing up the rear of every funeral I meet; "
|
||||
"and especially whenever my hypos get such an upper hand of me, that it "
|
||||
"requires a strong moral principle to prevent me from deliberately stepping "
|
||||
"into the street, and methodically knocking people's hats off—then, I account "
|
||||
"it high time to get to sea as soon as I can."
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
def _payload_bytes() -> bytes:
|
||||
"""Deterministic *Moby-Dick* EPUB used as the download payload.
|
||||
|
||||
Determinism matters: the webseed torrent's piece hashes are computed from these
|
||||
exact bytes, so any drift between what mock-aa serves and what the torrent
|
||||
describes would make qBittorrent never complete. Fixed ZipInfo timestamps keep
|
||||
the bytes byte-stable across runs.
|
||||
"""
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
files = [
|
||||
("mimetype", "application/epub+zip"),
|
||||
(
|
||||
"META-INF/container.xml",
|
||||
'<?xml version="1.0"?><container version="1.0" '
|
||||
'xmlns="urn:oasis:names:tc:opendocument:xmlns:container">'
|
||||
'<rootfiles><rootfile full-path="OEBPS/content.opf" '
|
||||
'media-type="application/oebps-package+xml"/></rootfiles></container>',
|
||||
),
|
||||
(
|
||||
"OEBPS/content.opf",
|
||||
'<?xml version="1.0"?><package xmlns="http://www.idpf.org/2007/opf" '
|
||||
'version="3.0" unique-identifier="id"><metadata '
|
||||
'xmlns:dc="http://purl.org/dc/elements/1.1/">'
|
||||
'<dc:identifier id="id">e2e-moby-dick</dc:identifier>'
|
||||
"<dc:title>Moby-Dick; or, The Whale</dc:title>"
|
||||
"<dc:creator>Herman Melville</dc:creator>"
|
||||
"<dc:language>en</dc:language></metadata>"
|
||||
'<manifest><item id="c1" href="chapter1.xhtml" '
|
||||
'media-type="application/xhtml+xml"/></manifest>'
|
||||
'<spine><itemref idref="c1"/></spine></package>',
|
||||
),
|
||||
(
|
||||
"OEBPS/chapter1.xhtml",
|
||||
'<?xml version="1.0" encoding="utf-8"?>'
|
||||
'<html xmlns="http://www.w3.org/1999/xhtml"><head><title>Loomings</title>'
|
||||
"</head><body><h1>Chapter 1. Loomings.</h1>"
|
||||
# Repeat the opening so the EPUB clears shelfmark's 10 KB minimum-size
|
||||
# check on the direct-download path (_MIN_VALID_FILE_SIZE).
|
||||
+ ("<p>" + _MOBY_DICK_TEXT + "</p>") * 24
|
||||
+ "</body></html>",
|
||||
),
|
||||
]
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf:
|
||||
for name, content in files:
|
||||
info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
|
||||
zf.writestr(info, content)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _read_fixture(name: str) -> str:
|
||||
return (FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Role: origin-aa (fake Anna's Archive)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _search_rows() -> str:
|
||||
"""One real result row in the exact shape the parser expects."""
|
||||
return _read_fixture("aa_search_results.html")
|
||||
|
||||
|
||||
def register_origin_aa(flask_app: Flask) -> None:
|
||||
@flask_app.route("/search")
|
||||
def aa_search() -> Response:
|
||||
query = request.args.get("q", "")
|
||||
# Fault injection travels INSIDE the search query (the app builds the AA
|
||||
# URL itself and won't forward arbitrary params), via a token the harness
|
||||
# embeds: "E2EINJECT:<name> <real query>".
|
||||
inject = ""
|
||||
match = re.search(r"E2EINJECT:(\w+)", query)
|
||||
if match:
|
||||
inject = match.group(1)
|
||||
query = re.sub(r"E2EINJECT:\w+\s*", "", query).strip()
|
||||
if inject == "500":
|
||||
return make_response("upstream error", 500)
|
||||
if inject == "empty":
|
||||
body = _read_fixture("aa_search_empty.html")
|
||||
return make_response(body, 200)
|
||||
if inject == "no_files":
|
||||
# Real row present *and* the "No files found." string — the historical
|
||||
# false-positive (#: 'No files found' check). The app must still
|
||||
# surface the real row.
|
||||
body = _search_rows().replace("<!-- NO_FILES_MARKER -->", "<div>No files found.</div>")
|
||||
return make_response(body, 200)
|
||||
if inject == "layout_drift":
|
||||
return make_response(_read_fixture("aa_search_layout_drift.html"), 200)
|
||||
# Echo the query into the title so tests can assert routing worked.
|
||||
body = _search_rows().replace("__QUERY__", query or "A Book Title")
|
||||
return make_response(body, 200)
|
||||
|
||||
@flask_app.route("/md5/<book_id>")
|
||||
def aa_detail(book_id: str) -> Response:
|
||||
# __SLOW_BASE__ controls where the AA "slow partner server" links point.
|
||||
# In the `full` profile it's the Cloudflare gate, so the *download* (not the
|
||||
# search/detail) is what forces the internal Chrome bypasser to solve CF.
|
||||
body = (
|
||||
_read_fixture("aa_detail.html")
|
||||
.replace("__MD5__", book_id)
|
||||
.replace("__SLOW_BASE__", SLOW_DOWNLOAD_BASE)
|
||||
)
|
||||
return make_response(body, 200)
|
||||
|
||||
@flask_app.route("/get.php")
|
||||
def aa_getphp() -> Response:
|
||||
# The actual file download link target (get.php?md5=..&key=..).
|
||||
return _serve_book()
|
||||
|
||||
@flask_app.route("/slow_download/<path:rest>")
|
||||
def aa_slow(rest: str) -> Response:
|
||||
# The AA "slow partner server" page: an HTML page whose "Download now" link
|
||||
# is the final file URL. shelfmark's _extract_slow_download_url parses this.
|
||||
# Reached (in the full profile) only after the internal Chrome bypasser
|
||||
# solves the Cloudflare gate in front of it.
|
||||
del rest
|
||||
file_url = f"{os.environ.get('AA_FILE_BASE', 'http://mock-aa')}/file/{PAYLOAD_NAME}"
|
||||
# The visible text must exceed the internal bypasser's "still loading"
|
||||
# threshold (_LOADING_BODY_LENGTH_MAX = 50 chars of body.innerText) or it
|
||||
# never considers the (cleared) page settled and loops until timeout.
|
||||
html = (
|
||||
"<!doctype html><html><head><title>Download</title></head><body>"
|
||||
"<h1>Anna’s Archive — Slow Partner Server</h1>"
|
||||
"<p>Your download of <em>Moby-Dick; or, The Whale</em> by Herman Melville "
|
||||
"is ready. Use the link below to download the file from this slow partner "
|
||||
"server. The connection is slow but free, with no waitlist.</p>"
|
||||
"<div class='top-row'>"
|
||||
f'<a href="{file_url}" download>\U0001f4da Download now</a>'
|
||||
"</div>"
|
||||
"<p>Thank you for supporting open access to knowledge.</p>"
|
||||
"</body></html>"
|
||||
)
|
||||
return make_response(html, 200)
|
||||
|
||||
@flask_app.route(f"/file/{PAYLOAD_NAME}")
|
||||
def aa_file() -> Response:
|
||||
# Final file URL extracted from the slow-download page.
|
||||
return _serve_book()
|
||||
|
||||
@flask_app.route("/dyn/api/fast_download.json")
|
||||
def aa_fast() -> Response:
|
||||
md5 = request.args.get("md5", "")
|
||||
return jsonify({"download_url": f"{request.host_url.rstrip('/')}/get.php?md5={md5}&key=k"})
|
||||
|
||||
# --- webseed payload + torrent for the `full` real-client pipeline --------
|
||||
@flask_app.route(f"/payload/{PAYLOAD_NAME}")
|
||||
def aa_payload() -> Response:
|
||||
# Range support is required for transmission's GetRight webseed (it fetches
|
||||
# pieces with `Range: bytes=...` and expects 206); libtorrent clients
|
||||
# (qBittorrent/deluge) tolerate a plain 200, but transmission does not.
|
||||
return _ranged_response(_payload_bytes(), "application/epub+zip")
|
||||
|
||||
@flask_app.route("/payload.torrent")
|
||||
def aa_torrent() -> Response:
|
||||
torrent = _build_payload_torrent()
|
||||
resp = make_response(torrent)
|
||||
resp.headers["Content-Type"] = "application/x-bittorrent"
|
||||
resp.headers["Content-Disposition"] = 'attachment; filename="sample-book.torrent"'
|
||||
return resp
|
||||
|
||||
|
||||
def _ranged_response(data: bytes, content_type: str) -> Response:
|
||||
"""Serve ``data`` honoring a single HTTP Range request (206 + Content-Range).
|
||||
|
||||
Needed so transmission's webseed (which fetches via ``Range: bytes=...``) can
|
||||
download piece by piece. A request without Range gets the full 200 body.
|
||||
"""
|
||||
total = len(data)
|
||||
range_header = request.headers.get("Range", "")
|
||||
if range_header.startswith("bytes="):
|
||||
first = range_header[len("bytes=") :].split(",", 1)[0]
|
||||
start_s, _, end_s = first.partition("-")
|
||||
try:
|
||||
start = int(start_s) if start_s else 0
|
||||
end = int(end_s) if end_s else total - 1
|
||||
except ValueError:
|
||||
start, end = 0, total - 1
|
||||
end = min(end, total - 1)
|
||||
start = max(0, min(start, end))
|
||||
chunk = data[start : end + 1]
|
||||
resp = make_response(chunk, 206)
|
||||
resp.headers["Content-Range"] = f"bytes {start}-{end}/{total}"
|
||||
else:
|
||||
resp = make_response(data)
|
||||
resp.headers["Content-Type"] = content_type
|
||||
resp.headers["Accept-Ranges"] = "bytes"
|
||||
return resp
|
||||
|
||||
|
||||
def _serve_book() -> Response:
|
||||
resp = _ranged_response(_payload_bytes(), "application/epub+zip")
|
||||
resp.headers["Content-Disposition"] = f'attachment; filename="{PAYLOAD_NAME}"'
|
||||
return resp
|
||||
|
||||
|
||||
def _build_payload_torrent() -> bytes:
|
||||
"""Webseed .torrent for the deterministic payload, sourced only from mock-aa.
|
||||
|
||||
Imported lazily so the doh/cloudflare/flaresolverr roles don't need the
|
||||
generator module on the path.
|
||||
"""
|
||||
import importlib.util
|
||||
|
||||
gen_path = Path(__file__).parent / "make_webseed_torrent.py"
|
||||
spec = importlib.util.spec_from_file_location("make_webseed_torrent", gen_path)
|
||||
assert spec and spec.loader
|
||||
gen = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(gen)
|
||||
return gen.build_webseed_torrent(PAYLOAD_NAME, _payload_bytes(), PAYLOAD_URL)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Role: cloudflare (challenge until clearance cookie present)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def register_cloudflare(flask_app: Flask) -> None:
|
||||
@flask_app.route("/", defaults={"path": ""})
|
||||
@flask_app.route("/<path:path>")
|
||||
def cf_gate(path: str) -> Response:
|
||||
if request.cookies.get(CLEARANCE_COOKIE) == CLEARANCE_VALUE:
|
||||
# Cleared: proxy the request through to the real AA origin behaviour.
|
||||
return _cleared_passthrough(path)
|
||||
challenge = _read_fixture("cf_challenge.html")
|
||||
resp = make_response(challenge, 403)
|
||||
resp.headers["cf-mitigated"] = "challenge"
|
||||
resp.headers["Server"] = "cloudflare"
|
||||
return resp
|
||||
|
||||
|
||||
def _cleared_passthrough(path: str) -> Response:
|
||||
import requests as _rq
|
||||
|
||||
target = f"{ORIGIN_INTERNAL_URL}/{path}"
|
||||
upstream = _rq.get(target, params=request.args, timeout=10)
|
||||
resp = make_response(upstream.content, upstream.status_code)
|
||||
resp.headers["Content-Type"] = upstream.headers.get("Content-Type", "text/html")
|
||||
return resp
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Role: flaresolverr (mock external bypasser, /v1 contract)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def register_flaresolverr(flask_app: Flask) -> None:
|
||||
@flask_app.route("/v1", methods=["POST"])
|
||||
def v1() -> Response:
|
||||
import requests as _rq
|
||||
|
||||
payload = request.get_json(silent=True) or {}
|
||||
url = payload.get("url", "")
|
||||
if not url:
|
||||
return jsonify({"status": "error", "message": "missing url"}), 400
|
||||
# "Solve" the challenge by fetching with the clearance cookie set.
|
||||
upstream = _rq.get(url, cookies={CLEARANCE_COOKIE: CLEARANCE_VALUE}, timeout=15)
|
||||
return jsonify(
|
||||
{
|
||||
"status": "ok",
|
||||
"message": "Challenge solved!",
|
||||
"solution": {
|
||||
"url": url,
|
||||
"status": upstream.status_code,
|
||||
"response": upstream.text,
|
||||
"cookies": [{"name": CLEARANCE_COOKIE, "value": CLEARANCE_VALUE, "domain": ""}],
|
||||
"userAgent": "Mozilla/5.0 (e2e-flaresolverr)",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Role: prowlarr (minimal Prowlarr API for the `full` real-client pipeline)
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Implements just the endpoints shelfmark's prowlarr client calls, returning one
|
||||
# torrent release whose download is mock-aa's webseed .torrent. A real
|
||||
# qBittorrent then completes the download over HTTP (no tracker/peer needed).
|
||||
AA_INTERNAL_URL = os.environ.get("AA_INTERNAL_URL", "http://mock-aa")
|
||||
|
||||
|
||||
def register_prowlarr(flask_app: Flask) -> None:
|
||||
@flask_app.route("/api/v1/system/status")
|
||||
def prowlarr_status() -> Response:
|
||||
return jsonify({"appName": "Prowlarr", "version": "1.30.0.4000", "instanceName": "e2e"})
|
||||
|
||||
@flask_app.route("/api/v1/indexer")
|
||||
def prowlarr_indexers() -> Response:
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Mock Torznab",
|
||||
"enable": True,
|
||||
"protocol": "torrent",
|
||||
"implementation": "Torznab",
|
||||
"implementationName": "Generic Torznab",
|
||||
"definitionName": "mock-torznab",
|
||||
"capabilities": {
|
||||
"categories": [
|
||||
{"id": 7000, "name": "Books"},
|
||||
{"id": 7020, "name": "Books/EBook"},
|
||||
]
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
@flask_app.route("/api/v1/indexer/<int:indexer_id>/newznab")
|
||||
def prowlarr_torznab(indexer_id: int) -> Response:
|
||||
del indexer_id
|
||||
t = request.args.get("t", "search")
|
||||
if t == "caps":
|
||||
return Response(_torznab_caps(), mimetype="application/xml")
|
||||
query = request.args.get("q", "") or "E2E Mock Book"
|
||||
return Response(_torznab_search(query), mimetype="application/xml")
|
||||
|
||||
|
||||
def _torznab_caps() -> str:
|
||||
return (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<caps><server title="Mock Torznab"/>'
|
||||
'<limits max="100" default="50"/>'
|
||||
'<searching><search available="yes" supportedParams="q"/>'
|
||||
'<book-search available="yes" supportedParams="q,author,title"/></searching>'
|
||||
'<categories><category id="7000" name="Books">'
|
||||
'<subcat id="7020" name="EBook"/></category></categories></caps>'
|
||||
)
|
||||
|
||||
|
||||
def _torznab_search(query: str) -> str:
|
||||
torrent_url = f"{AA_INTERNAL_URL}/payload.torrent"
|
||||
size = len(_payload_bytes())
|
||||
title = f"{query} - E2E Mock Book"
|
||||
return (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<rss version="2.0" xmlns:torznab="http://torznab.com/schemas/2015/feed">'
|
||||
"<channel>"
|
||||
"<item>"
|
||||
f"<title>{title}</title>"
|
||||
"<guid>e2e-mock-release-1</guid>"
|
||||
f"<link>{torrent_url}</link>"
|
||||
f"<size>{size}</size>"
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 +0000</pubDate>"
|
||||
f'<enclosure url="{torrent_url}" length="{size}" type="application/x-bittorrent"/>'
|
||||
'<torznab:attr name="category" value="7020"/>'
|
||||
'<torznab:attr name="seeders" value="10"/>'
|
||||
'<torznab:attr name="peers" value="11"/>'
|
||||
'<torznab:attr name="downloadvolumefactor" value="0"/>'
|
||||
'<torznab:attr name="uploadvolumefactor" value="1"/>'
|
||||
"</item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Role: doh (DNS over HTTPS responder)
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Maps mock hostnames to the in-network IP the test wants them resolved to.
|
||||
# Provided via DOH_MAP env: "host=ip,host2=ip2".
|
||||
def _doh_map() -> dict[str, str]:
|
||||
raw = os.environ.get("DOH_MAP", "")
|
||||
out: dict[str, str] = {}
|
||||
for pair in raw.split(","):
|
||||
pair = pair.strip()
|
||||
if "=" in pair:
|
||||
host, ip = pair.split("=", 1)
|
||||
out[host.strip().rstrip(".").lower()] = ip.strip()
|
||||
return out
|
||||
|
||||
|
||||
def _encode_a_answer(name: str, ip: str) -> bytes:
|
||||
parts = ip.split(".")
|
||||
return struct.pack("!HHHIH4B", 0xC00C, 1, 1, 60, 4, *(int(p) for p in parts))
|
||||
|
||||
|
||||
def register_doh(flask_app: Flask) -> None:
|
||||
@flask_app.route("/dns-query", methods=["GET", "POST"])
|
||||
@flask_app.route("/resolve", methods=["GET", "POST"]) # Google-style JSON endpoint
|
||||
def dns_query() -> Response:
|
||||
mapping = _doh_map()
|
||||
# Google/Cloudflare JSON form (?name=&type=A)
|
||||
name = (request.args.get("name") or "").rstrip(".").lower()
|
||||
if name:
|
||||
ip = mapping.get(name)
|
||||
answer = [{"name": name, "type": 1, "TTL": 60, "data": ip}] if ip else []
|
||||
return jsonify({"Status": 0 if ip else 3, "Answer": answer})
|
||||
# RFC 8484 wireformat (POST body or ?dns=)
|
||||
if request.method == "POST":
|
||||
wire = request.get_data()
|
||||
else:
|
||||
dns_b64 = request.args.get("dns", "")
|
||||
wire = base64.urlsafe_b64decode(dns_b64 + "=" * (-len(dns_b64) % 4))
|
||||
return _wireformat_response(wire, mapping)
|
||||
|
||||
def _wireformat_response(wire: bytes, mapping: dict[str, str]) -> Response:
|
||||
# Minimal parser: echo header/question, append one A answer if known.
|
||||
txid = wire[0:2]
|
||||
qname, _ = _parse_qname(wire, 12)
|
||||
question = wire[12:]
|
||||
host = qname.rstrip(".").lower()
|
||||
ip = mapping.get(host)
|
||||
ancount = 1 if ip else 0
|
||||
header = txid + struct.pack("!HHHHH", 0x8180, 1, ancount, 0, 0)
|
||||
body = question + (_encode_a_answer(host, ip) if ip else b"")
|
||||
resp = make_response(header + body)
|
||||
resp.headers["Content-Type"] = "application/dns-message"
|
||||
return resp
|
||||
|
||||
def _parse_qname(wire: bytes, offset: int) -> tuple[str, int]:
|
||||
labels = []
|
||||
while True:
|
||||
length = wire[offset]
|
||||
offset += 1
|
||||
if length == 0:
|
||||
break
|
||||
labels.append(wire[offset : offset + length].decode("ascii", "ignore"))
|
||||
offset += length
|
||||
return ".".join(labels), offset
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Health + role wiring
|
||||
# --------------------------------------------------------------------------- #
|
||||
@app.route("/healthz")
|
||||
def healthz() -> Response:
|
||||
return jsonify({"role": ROLE, "ok": True})
|
||||
|
||||
|
||||
_ROLES = {
|
||||
"origin-aa": register_origin_aa,
|
||||
"cloudflare": register_cloudflare,
|
||||
"flaresolverr": register_flaresolverr,
|
||||
"prowlarr": register_prowlarr,
|
||||
"doh": register_doh,
|
||||
}
|
||||
|
||||
if ROLE == "all":
|
||||
for _register in _ROLES.values():
|
||||
_register(app)
|
||||
elif ROLE in _ROLES:
|
||||
_ROLES[ROLE](app)
|
||||
else: # pragma: no cover - misconfiguration guard
|
||||
raise SystemExit(f"Unknown MOCK_ROLE={ROLE!r}; expected one of {[*sorted(_ROLES), 'all']}")
|
||||
|
||||
|
||||
def _self_signed_cert() -> tuple[str, str]:
|
||||
"""Write a throwaway self-signed cert/key to /tmp and return their paths.
|
||||
|
||||
Used only by the `doh` role's HTTPS server. The app reaches it with
|
||||
CERTIFICATE_VALIDATION=disabled, so the cert's identity is irrelevant.
|
||||
"""
|
||||
import datetime
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "e2e-doh")])
|
||||
now = datetime.datetime.now(datetime.UTC)
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name)
|
||||
.issuer_name(name)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - datetime.timedelta(days=1))
|
||||
.not_valid_after(now + datetime.timedelta(days=3650))
|
||||
.add_extension(x509.SubjectAlternativeName([x509.DNSName("cloudflare-dns.com")]), False)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
cert_path, key_path = "/tmp/doh.crt", "/tmp/doh.key"
|
||||
Path(cert_path).write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
Path(key_path).write_bytes(
|
||||
key.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
)
|
||||
return cert_path, key_path
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.environ.get("DOH_TLS") == "1":
|
||||
cert_path, key_path = _self_signed_cert()
|
||||
app.run(host="0.0.0.0", port=443, ssl_context=(cert_path, key_path), threaded=True)
|
||||
else:
|
||||
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", "80")))
|
||||
@@ -0,0 +1,4 @@
|
||||
flask>=3.0
|
||||
requests>=2.31
|
||||
# Only used by the `doh` role when DOH_TLS=1, to self-sign a cert at startup.
|
||||
cryptography>=42.0
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
[Application]
|
||||
FileLogger\Enabled=true
|
||||
|
||||
[BitTorrent]
|
||||
Session\DefaultSavePath=/downloads
|
||||
Session\TempPathEnabled=false
|
||||
# Webseed (BEP-19) is how the e2e torrent completes with no tracker/peer.
|
||||
|
||||
[LegalNotice]
|
||||
Accepted=true
|
||||
|
||||
[Preferences]
|
||||
WebUI\Address=*
|
||||
WebUI\Port=8080
|
||||
# Bypass WebUI auth for the e2e compose subnet so shelfmark connects without the
|
||||
# linuxserver image's random temp password.
|
||||
WebUI\LocalHostAuth=false
|
||||
WebUI\AuthSubnetWhitelistEnabled=true
|
||||
WebUI\AuthSubnetWhitelist=172.30.0.0/24
|
||||
WebUI\CSRFProtection=false
|
||||
WebUI\HostHeaderValidation=false
|
||||
Downloads\SavePath=/downloads/
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the Shelfmark e2e platform for a single config profile.
|
||||
#
|
||||
# ./run-e2e.sh [env/<profile>.env] [extra pytest args...]
|
||||
#
|
||||
# Boots the stack defined by the profile env file, waits for health, runs the
|
||||
# matching cluster tests (the suite skips tests not applicable to the profile),
|
||||
# then tears down. Set KEEP_UP=1 to leave the stack running for debugging.
|
||||
set -euo pipefail
|
||||
|
||||
PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$PLATFORM_DIR"
|
||||
|
||||
ENV_FILE="${1:-env/baseline.env}"
|
||||
shift || true
|
||||
PYTEST_ARGS=("$@")
|
||||
|
||||
if [[ ! -f "$ENV_FILE" ]]; then
|
||||
echo "error: env file not found: $ENV_FILE" >&2
|
||||
echo "available profiles:" >&2
|
||||
ls env/*.env >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
set -a; source "$ENV_FILE"; set +a # export SM_*, COMPOSE_PROFILES, E2E_PROFILE
|
||||
PROFILE="${E2E_PROFILE:-baseline}"
|
||||
if [[ "$PROFILE" == "client-qbittorrent-delayed" ]]; then
|
||||
E2E_RUN_ID="${E2E_RUN_ID:-$(date +%s)-$$}"
|
||||
export SM_DOWNLOADS_HOST_DIR="${SM_DOWNLOADS_HOST_DIR:-./.state/$PROFILE/$E2E_RUN_ID/shelfmark-downloads}"
|
||||
export SM_QBITTORRENT_DOWNLOADS_HOST_DIR="${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/$PROFILE/$E2E_RUN_ID/client-downloads}"
|
||||
mkdir -p "$SM_DOWNLOADS_HOST_DIR" "$SM_QBITTORRENT_DOWNLOADS_HOST_DIR"
|
||||
fi
|
||||
COMPOSE=(docker compose --env-file "$ENV_FILE" -f docker-compose.e2e.yml)
|
||||
|
||||
STATE_DIR="$PLATFORM_DIR/.state"
|
||||
LOG_FILE="$STATE_DIR/shelfmark.$PROFILE.log"
|
||||
mkdir -p "$STATE_DIR/config" "$STATE_DIR/books" "$STATE_DIR/downloads" "$STATE_DIR/tmp"
|
||||
|
||||
cleanup() {
|
||||
if [[ "${KEEP_UP:-0}" != "1" ]]; then
|
||||
echo "==> tearing down ($PROFILE)"
|
||||
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
|
||||
else
|
||||
echo "==> KEEP_UP=1: leaving stack running ($PROFILE)"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# E2E_NO_BUILD=1 reuses already-built images (see `make e2e-platform-build` /
|
||||
# run-matrix.sh) so a matrix run builds the heavy shelfmark image only once.
|
||||
if [[ "${E2E_NO_BUILD:-0}" == "1" ]]; then
|
||||
echo "==> [$PROFILE] starting stack, reusing built images (profiles='${COMPOSE_PROFILES:-<none>}')"
|
||||
"${COMPOSE[@]}" up -d --no-build
|
||||
else
|
||||
echo "==> [$PROFILE] building + starting stack (profiles='${COMPOSE_PROFILES:-<none>}')"
|
||||
"${COMPOSE[@]}" up -d --build
|
||||
fi
|
||||
|
||||
echo "==> [$PROFILE] waiting for shelfmark health"
|
||||
HEALTHY=0
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS http://localhost:8084/api/health >/dev/null 2>&1; then HEALTHY=1; break; fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Capture boot diagnostics for the entrypoint/permission tests.
|
||||
"${COMPOSE[@]}" logs shelfmark > "$LOG_FILE" 2>&1 || true
|
||||
RESTARTS="$(docker inspect -f '{{.RestartCount}}' e2e-shelfmark 2>/dev/null || echo 0)"
|
||||
echo "==> [$PROFILE] healthy=$HEALTHY restarts=$RESTARTS log=$LOG_FILE"
|
||||
|
||||
if [[ "$HEALTHY" != "1" && "$PROFILE" != "tor" ]]; then
|
||||
echo "error: shelfmark never became healthy under profile '$PROFILE'" >&2
|
||||
"${COMPOSE[@]}" logs --tail 40 shelfmark >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Hand context to the pytest suite.
|
||||
export E2E_PROFILE="$PROFILE"
|
||||
export E2E_BASE_URL="http://localhost:8084"
|
||||
export E2E_BOOKS_DIR="$STATE_DIR/books"
|
||||
export E2E_TMP_DIR="$STATE_DIR/tmp"
|
||||
export E2E_SHELFMARK_LOG="$LOG_FILE"
|
||||
export E2E_SHELFMARK_RESTARTS="$RESTARTS"
|
||||
|
||||
echo "==> [$PROFILE] running suite"
|
||||
set +e
|
||||
( cd "$PLATFORM_DIR/../../.." && \
|
||||
uv run pytest tests/e2e/platform/suite -m platform -o addopts="--tb=short" "${PYTEST_ARGS[@]}" )
|
||||
RC=$?
|
||||
set -e
|
||||
|
||||
echo "==> [$PROFILE] pytest exit=$RC"
|
||||
exit $RC
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the full config matrix: every profile, in sequence, aggregating results.
|
||||
#
|
||||
# ./run-matrix.sh # all profiles
|
||||
# ./run-matrix.sh baseline dns-manual # a subset (by profile name)
|
||||
set -uo pipefail
|
||||
|
||||
PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$PLATFORM_DIR"
|
||||
|
||||
if [[ $# -gt 0 ]]; then
|
||||
PROFILES=("$@")
|
||||
else
|
||||
PROFILES=(baseline bypasser-external bypasser-disabled dns-manual dns-blocked dns-doh \
|
||||
proxy-http proxy-socks tor client-transmission client-deluge \
|
||||
client-qbittorrent-delayed)
|
||||
fi
|
||||
|
||||
# Build the (heavy) images once, then reuse them across every profile so the
|
||||
# matrix doesn't rebuild the xvfb/chromium layer N times. Set NO_PREBUILD=1 to
|
||||
# skip (e.g. to let each run rebuild from source).
|
||||
if [[ "${NO_PREBUILD:-0}" != "1" ]]; then
|
||||
echo "==> pre-building images once (reused by all profiles)"
|
||||
./build-images.sh
|
||||
export E2E_NO_BUILD=1
|
||||
fi
|
||||
|
||||
declare -A RESULT
|
||||
FAILED=0
|
||||
for p in "${PROFILES[@]}"; do
|
||||
echo "========================================================================"
|
||||
echo " PROFILE: $p"
|
||||
echo "========================================================================"
|
||||
if ./run-e2e.sh "env/$p.env"; then
|
||||
RESULT[$p]="PASS"
|
||||
else
|
||||
RESULT[$p]="FAIL"
|
||||
FAILED=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "========================================================================"
|
||||
echo " MATRIX SUMMARY"
|
||||
echo "========================================================================"
|
||||
for p in "${PROFILES[@]}"; do
|
||||
printf " %-22s %s\n" "$p" "${RESULT[$p]:-SKIP}"
|
||||
done
|
||||
exit $FAILED
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Harness for the Shelfmark e2e Docker platform.
|
||||
|
||||
These tests run against a *live* Shelfmark booted by ``run-e2e.sh`` under a
|
||||
particular config profile (env file). The active profile is read from
|
||||
``E2E_PROFILE``; tests select which profiles they apply to with the
|
||||
``@pytest.mark.profiles(...)`` marker. Unmarked tests are profile-agnostic
|
||||
invariants and run under every profile — that is how the same cluster test
|
||||
becomes the config matrix (search must succeed whether egress is direct, via a
|
||||
proxy, via custom DNS, or through the bypasser).
|
||||
|
||||
Run (handled by run-e2e.sh):
|
||||
E2E_PROFILE=baseline uv run pytest tests/e2e/platform/suite -m platform
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
# This conftest's pytest_collection_modifyitems hook receives the *whole*
|
||||
# session's items (not just ones under this dir), so scope our marking to the
|
||||
# suite to avoid tagging the entire repo's tests as platform/e2e.
|
||||
_SUITE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
BASE_URL = os.environ.get("E2E_BASE_URL", "http://localhost:8084")
|
||||
ACTIVE_PROFILE = os.environ.get("E2E_PROFILE", "baseline")
|
||||
DEFAULT_TIMEOUT = 15
|
||||
DOWNLOAD_TIMEOUT = int(os.environ.get("E2E_DOWNLOAD_TIMEOUT", "120"))
|
||||
TERMINAL_OK = {"complete", "done", "available"}
|
||||
TERMINAL_ERR = {"error", "cancelled"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlatformClient:
|
||||
base_url: str = BASE_URL
|
||||
timeout: int = DEFAULT_TIMEOUT
|
||||
session: requests.Session = field(default_factory=requests.Session)
|
||||
|
||||
def get(self, path: str, **kw) -> requests.Response:
|
||||
kw.setdefault("timeout", self.timeout)
|
||||
return self.session.get(f"{self.base_url}{path}", **kw)
|
||||
|
||||
def post(self, path: str, **kw) -> requests.Response:
|
||||
kw.setdefault("timeout", self.timeout)
|
||||
return self.session.post(f"{self.base_url}{path}", **kw)
|
||||
|
||||
# --- domain helpers -------------------------------------------------- #
|
||||
def wait_for_health(self, max_wait: int = 90) -> bool:
|
||||
deadline = time.time() + max_wait
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if self.get("/api/health").status_code == 200:
|
||||
return True
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(2)
|
||||
return False
|
||||
|
||||
def direct_search(
|
||||
self, query: str, *, inject: str | None = None, **params
|
||||
) -> requests.Response:
|
||||
"""Source-native (hermetic) release search — no external metadata provider.
|
||||
|
||||
Hits GET /api/releases?source=direct_download&query=... which drives
|
||||
direct_download.search_books against the mock Anna's Archive.
|
||||
|
||||
Fault injection rides *inside* the query text (the app builds the AA URL
|
||||
itself and only forwards the query as ``q=``); the mock origin parses the
|
||||
``E2EINJECT:<name>`` token. See mock_services.aa_search.
|
||||
"""
|
||||
effective_query = f"E2EINJECT:{inject} {query}" if inject else query
|
||||
qp = {"source": "direct_download", "query": effective_query, **params}
|
||||
return self.get("/api/releases", params=qp, timeout=60)
|
||||
|
||||
def releases_from(self, resp: requests.Response) -> list[dict]:
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
data = resp.json()
|
||||
if isinstance(data, dict):
|
||||
rel = data.get("releases")
|
||||
return rel if isinstance(rel, list) else []
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def queue_download(self, release: dict) -> requests.Response:
|
||||
return self.post("/api/releases/download", json=release, timeout=30)
|
||||
|
||||
def wait_for_terminal(self, book_id: str, timeout: int = DOWNLOAD_TIMEOUT) -> tuple[str, dict]:
|
||||
deadline = time.time() + timeout
|
||||
last: dict = {}
|
||||
while time.time() < deadline:
|
||||
resp = self.get("/api/status")
|
||||
if resp.status_code == 200 and isinstance(resp.json(), dict):
|
||||
status = resp.json()
|
||||
for state, entries in status.items():
|
||||
if isinstance(entries, dict) and book_id in entries:
|
||||
last = entries[book_id]
|
||||
if state in TERMINAL_OK or state in TERMINAL_ERR:
|
||||
return state, last
|
||||
time.sleep(2)
|
||||
return "timeout", last
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client() -> PlatformClient:
|
||||
c = PlatformClient()
|
||||
if not c.wait_for_health():
|
||||
pytest.fail(f"Shelfmark not healthy at {BASE_URL} (profile={ACTIVE_PROFILE})")
|
||||
return c
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def active_profile() -> str:
|
||||
return ACTIVE_PROFILE
|
||||
|
||||
|
||||
def pytest_configure(config: pytest.Config) -> None:
|
||||
config.addinivalue_line("markers", "platform: Shelfmark e2e docker platform test")
|
||||
config.addinivalue_line("markers", "profiles(*names): only run under these E2E_PROFILE values")
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||
"""Skip tests whose declared profiles don't include the active profile.
|
||||
|
||||
A test with no ``profiles`` marker is a profile-agnostic invariant and runs
|
||||
everywhere (this is the matrix: invariants x profiles).
|
||||
"""
|
||||
for item in items:
|
||||
if not item.path.is_relative_to(_SUITE_DIR):
|
||||
continue
|
||||
item.add_marker(pytest.mark.platform)
|
||||
item.add_marker(pytest.mark.e2e)
|
||||
marker = item.get_closest_marker("profiles")
|
||||
if marker and ACTIVE_PROFILE not in marker.args:
|
||||
item.add_marker(
|
||||
pytest.mark.skip(reason=f"profile={ACTIVE_PROFILE!r} not in {marker.args}")
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Cluster 1 — Cloudflare bypasser wiring + clean-failure behavior.
|
||||
|
||||
Reality discovered by running the stack: shelfmark's AA *search* and *detail*
|
||||
fetches use ``html_get_page(allow_bypasser_fallback=False)``, so a search behind a
|
||||
Cloudflare gate returns 503 **regardless** of the bypasser. The bypasser (internal
|
||||
Chrome or external FlareSolverr) is a *download-time* mechanism
|
||||
(``html_get_page(use_bypasser=True)``); it never runs for search.
|
||||
|
||||
So these tests assert what is actually true and host-observable:
|
||||
* the external bypasser is configured from env, and
|
||||
* a CF-gated search fails *cleanly* (a 503 the client can act on, not a hang or
|
||||
a crash) — both with the bypasser on (it isn't used for search) and off.
|
||||
|
||||
Exercising shelfmark's *use* of the bypasser end-to-end (a real CF solve during a
|
||||
download) needs the AA slow-download HTML flow mocked — see the README roadmap.
|
||||
The bypass *mechanism* itself is verified to work: the mock FlareSolverr solves
|
||||
the gate (manually confirmed; see README). Guards: #284 #226 #202 #1030 #410 #369.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
|
||||
def _boot_log() -> str:
|
||||
path = os.environ.get("E2E_SHELFMARK_LOG")
|
||||
if not path or not Path(path).exists():
|
||||
return ""
|
||||
return Path(path).read_text(encoding="utf-8", errors="ignore")
|
||||
|
||||
|
||||
def _cf_gated_search_has_no_releases(client) -> bool:
|
||||
"""A CF-gated AA search must not yield releases (the gate isn't bypassed for
|
||||
search).
|
||||
|
||||
NOTE (observed live): with ``USE_CF_BYPASS=false`` the search is *slow* to fail
|
||||
— the app retries and can take ~60s, vs a fast 503 when the bypasser is enabled
|
||||
(potential #1001 "hung on bypass protection"). We bound the wait and treat a
|
||||
timeout the same as a clean failure: in both cases no releases were obtained,
|
||||
which is the point of this negative control.
|
||||
"""
|
||||
try:
|
||||
resp = client.get(
|
||||
"/api/releases",
|
||||
params={"source": "direct_download", "query": "Mistborn"},
|
||||
timeout=30,
|
||||
)
|
||||
except requests.exceptions.Timeout:
|
||||
return True # could not complete -> definitively no releases obtained
|
||||
assert resp.status_code in (200, 404, 500, 503), (
|
||||
f"CF-gated search returned an unexpected status: {resp.status_code} {resp.text[:200]}"
|
||||
)
|
||||
return not client.releases_from(resp)
|
||||
|
||||
|
||||
@pytest.mark.profiles("bypasser-external")
|
||||
def test_external_bypasser_is_configured(client) -> None:
|
||||
"""The external (FlareSolverr) bypasser path is selected via env."""
|
||||
assert client.get("/api/health").status_code == 200
|
||||
log = _boot_log()
|
||||
if not log:
|
||||
pytest.skip("E2E_SHELFMARK_LOG not available")
|
||||
assert "USING_EXTERNAL_BYPASSER" in log and "EXT_BYPASSER_URL" in log, (
|
||||
"external bypasser config was not synced from env"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.profiles("bypasser-external")
|
||||
def test_cf_gated_search_fails_cleanly_with_external_bypasser(client) -> None:
|
||||
"""Even with the external bypasser configured, a CF-gated *search* yields no
|
||||
releases (the bypasser is download-time) — but it must fail cleanly."""
|
||||
assert _cf_gated_search_has_no_releases(client)
|
||||
|
||||
|
||||
@pytest.mark.profiles("bypasser-disabled")
|
||||
def test_cf_gated_search_fails_when_bypasser_off(client) -> None:
|
||||
"""Negative control: AA behind Cloudflare + bypasser OFF -> no releases, clean
|
||||
failure. A regression that ignored the gate would wrongly return results."""
|
||||
assert _cf_gated_search_has_no_releases(client), (
|
||||
"results returned even though AA is Cloudflare-gated and the bypasser is "
|
||||
"disabled — the challenge is being ignored (regression for #202/#410)"
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Cluster 1/6 — Tor boot correctness.
|
||||
|
||||
Tor has repeatedly boot-looped or pegged CPU on startup (#1021 loops on 1.3.0,
|
||||
#940 USING_TOR loop, #801 gosu 100% CPU, #937 gunicorn missing). The hermetic,
|
||||
fast assertion is: with USING_TOR=true the container reaches a healthy
|
||||
/api/health and does NOT crash-loop. Real Tor egress (slow/flaky in CI) is left
|
||||
to an opt-in 'tor-full' profile.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.profiles("tor")
|
||||
|
||||
|
||||
def test_app_becomes_healthy_under_tor(client) -> None:
|
||||
"""tor.sh + entrypoint must bring the app up, not boot-loop."""
|
||||
assert client.get("/api/health").status_code == 200
|
||||
|
||||
|
||||
def test_container_did_not_crash_loop(client) -> None:
|
||||
"""Restart count is captured by the runner into E2E_SHELFMARK_RESTARTS.
|
||||
|
||||
A boot-loop shows up as repeated restarts; a healthy boot is 0.
|
||||
"""
|
||||
restarts = os.environ.get("E2E_SHELFMARK_RESTARTS")
|
||||
if restarts is None:
|
||||
pytest.skip("runner did not provide E2E_SHELFMARK_RESTARTS")
|
||||
assert int(restarts) == 0, f"shelfmark restarted {restarts} times under Tor (boot-loop)"
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Cluster 5 — real download clients (torrent), end to end.
|
||||
|
||||
The `full` and `client-*` profiles each point shelfmark at a *real* torrent client
|
||||
(qBittorrent / Transmission / Deluge / rTorrent) plus a mock Prowlarr that returns a
|
||||
tracker-less BEP-19 webseed `.torrent` sourced from mock-aa. The client downloads
|
||||
the payload over HTTP (no tracker/peer/seeder) and shelfmark's completion detection
|
||||
+ post-process move lands Moby-Dick in `/books`.
|
||||
|
||||
The test is **client-agnostic** — the active profile's env selects the client
|
||||
(`PROWLARR_TORRENT_CLIENT` + that client's URL/creds) — so one test covers the whole
|
||||
client matrix.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.profiles(
|
||||
"full",
|
||||
"client-transmission",
|
||||
"client-deluge",
|
||||
"client-qbittorrent-delayed",
|
||||
)
|
||||
|
||||
BOOK = "Moby Dick"
|
||||
|
||||
|
||||
def _books_dir() -> Path | None:
|
||||
raw = os.environ.get("E2E_BOOKS_DIR")
|
||||
return Path(raw) if raw else None
|
||||
|
||||
|
||||
def _book_files(books: Path) -> set[str]:
|
||||
return {
|
||||
p.name for p in books.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}
|
||||
}
|
||||
|
||||
|
||||
def _prowlarr_search(client, query: str):
|
||||
return client.get(
|
||||
"/api/releases",
|
||||
params={
|
||||
"provider": "manual",
|
||||
"book_id": "e2e-manual-1",
|
||||
"source": "prowlarr",
|
||||
"title": query,
|
||||
"manual_query": query,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def test_prowlarr_to_real_torrent_client_download(client, active_profile) -> None:
|
||||
"""Prowlarr release -> real torrent client (per profile) -> file in /books."""
|
||||
books = _books_dir()
|
||||
if books is None or not books.exists():
|
||||
pytest.skip("E2E_BOOKS_DIR not visible to the test runner")
|
||||
before = _book_files(books)
|
||||
|
||||
resp = _prowlarr_search(client, BOOK)
|
||||
assert resp.status_code == 200, f"[{active_profile}] prowlarr search failed: {resp.text[:300]}"
|
||||
releases = client.releases_from(resp)
|
||||
assert releases, f"[{active_profile}] mock prowlarr returned no releases"
|
||||
|
||||
queued = client.queue_download(releases[0])
|
||||
assert queued.status_code in (200, 201, 202), (
|
||||
f"[{active_profile}] queue refused the release: {queued.status_code} {queued.text[:300]}"
|
||||
)
|
||||
# The prowlarr source serializes download_url=None and resolves the real URL
|
||||
# from its cache by source_id at download time.
|
||||
book_id = releases[0].get("source_id") or releases[0].get("id") or releases[0].get("guid")
|
||||
assert book_id, f"[{active_profile}] release missing a trackable id: {releases[0]!r}"
|
||||
|
||||
state, info = client.wait_for_terminal(str(book_id))
|
||||
assert state in {"complete", "done", "available"}, (
|
||||
f"[{active_profile}] real torrent-client download did not complete: "
|
||||
f"state={state} info={info!r}"
|
||||
)
|
||||
|
||||
deadline = time.time() + 30
|
||||
new_files: set[str] = set()
|
||||
while time.time() < deadline:
|
||||
new_files = _book_files(books) - before
|
||||
if new_files:
|
||||
break
|
||||
time.sleep(2)
|
||||
assert new_files, f"[{active_profile}] client completed but no file landed in /books"
|
||||
assert all(Path(n).suffix for n in new_files), f"file without extension: {new_files}"
|
||||
|
||||
if active_profile == "client-qbittorrent-delayed":
|
||||
_assert_completed_path_wait_engaged()
|
||||
|
||||
|
||||
def _assert_completed_path_wait_engaged() -> None:
|
||||
"""Confirm the delayed-sync profile actually exercised the path wait loop."""
|
||||
if shutil.which("docker") is None:
|
||||
return
|
||||
result = subprocess.run(
|
||||
["docker", "logs", "e2e-shelfmark"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
blob = (result.stdout + result.stderr).lower()
|
||||
if not blob.strip():
|
||||
return
|
||||
assert "completed files not available yet" in blob, (
|
||||
"delayed qBittorrent profile completed, but Shelfmark logs did not show "
|
||||
"the completed-path wait loop; the test may not have exercised #861"
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Cluster 4 — download execution + file placement/permissions.
|
||||
|
||||
Two halves, by what each profile can hermetically prove:
|
||||
|
||||
* **baseline (no bypasser, AA-only):** AA's slow-download sources require a
|
||||
Cloudflare bypass (``_CF_BYPASS_REQUIRED``), so a download here *cannot* succeed
|
||||
and the app must say so cleanly — this is exactly the real-world #1028 shape
|
||||
("All download sources failed"). We assert that the failure is surfaced as a
|
||||
terminal ``error`` with a message, not a hang/crash, and that nothing is left
|
||||
orphaned in staging (#1040).
|
||||
* **successful download + file move** (extension preserved #214, no orphaned
|
||||
staging dir #1040) is proven for real in the ``full`` profile, where a real
|
||||
qBittorrent completes a webseed torrent — see ``test_cluster_full_pipeline.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.profiles("baseline")
|
||||
|
||||
|
||||
def _staging_leftovers() -> list[Path]:
|
||||
tmp_raw = os.environ.get("E2E_TMP_DIR")
|
||||
if not tmp_raw or not Path(tmp_raw).exists():
|
||||
return []
|
||||
tmp = Path(tmp_raw)
|
||||
return [p for p in tmp.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}]
|
||||
|
||||
|
||||
def test_no_bypass_download_fails_cleanly(client) -> None:
|
||||
"""Without a bypasser, AA is undownloadable — the app must report a clear
|
||||
terminal error (the #1028 shape), not hang or crash."""
|
||||
resp = client.direct_search("Mistborn")
|
||||
releases = client.releases_from(resp)
|
||||
assert releases, "search should still return parsed releases even if undownloadable"
|
||||
|
||||
queued = client.queue_download(releases[0])
|
||||
assert queued.status_code in (200, 201, 202), (
|
||||
f"queue refused the release: {queued.status_code} {queued.text[:300]}"
|
||||
)
|
||||
book_id = releases[0].get("id") or releases[0].get("source_id") or releases[0].get("md5")
|
||||
assert book_id, f"release missing an id to track: {releases[0]!r}"
|
||||
|
||||
state, info = client.wait_for_terminal(str(book_id))
|
||||
assert state == "error", (
|
||||
f"expected a clean terminal error without a bypasser, got state={state} info={info!r}"
|
||||
)
|
||||
message = str(info.get("status_message") or info.get("last_error_message") or "")
|
||||
assert message.strip(), "download failed but surfaced no status message to the user"
|
||||
|
||||
|
||||
def test_no_orphaned_staging_dir(client) -> None:
|
||||
"""#1040 guard: a failed/aborted download must not leave book payloads behind
|
||||
in the staging/tmp area."""
|
||||
if not os.environ.get("E2E_TMP_DIR"):
|
||||
pytest.skip("E2E_TMP_DIR not provided")
|
||||
leftovers = _staging_leftovers()
|
||||
assert not leftovers, f"book payload left orphaned in staging dir: {leftovers}"
|
||||
@@ -0,0 +1,146 @@
|
||||
"""The `full` profile — maximum-realism, heavy, nightly/manual only.
|
||||
|
||||
Test book: *Moby-Dick* by Herman Melville (public domain). Validated live.
|
||||
|
||||
* **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are
|
||||
reachable (mock-aa), but the AA *slow-download* links point through the
|
||||
Cloudflare gate (mock-cf). Downloading therefore forces the in-image headless
|
||||
Chromium (seleniumbase CDP internal bypasser) to execute the challenge JS,
|
||||
harvest ``cf_clearance``, and fetch the gated slow-download page. Moby-Dick
|
||||
landing in ``/books`` is only possible if Chrome actually solved the gate —
|
||||
that is the literal "spin a chrome browser" path.
|
||||
* **DoH** enabled at boot (``USE_DOH=true``) without breaking startup.
|
||||
|
||||
The real torrent-client download (the other half of the `full` profile) lives in
|
||||
test_cluster_clients.py, which runs under `full` and the `client-*` profiles.
|
||||
Only runs under the ``full`` profile booted by ``run-e2e.sh env/full.env``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.profiles("full")
|
||||
|
||||
BOOK = "Moby Dick"
|
||||
|
||||
|
||||
def _boot_log() -> str:
|
||||
path = os.environ.get("E2E_SHELFMARK_LOG")
|
||||
if not path or not Path(path).exists():
|
||||
return ""
|
||||
return Path(path).read_text(encoding="utf-8", errors="ignore")
|
||||
|
||||
|
||||
def _books_dir() -> Path | None:
|
||||
raw = os.environ.get("E2E_BOOKS_DIR")
|
||||
return Path(raw) if raw else None
|
||||
|
||||
|
||||
def _book_files(books: Path) -> set[str]:
|
||||
return {
|
||||
p.name for p in books.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}
|
||||
}
|
||||
|
||||
|
||||
def _wait_for_new_book(books: Path, before: set[str], timeout: int = 40) -> set[str]:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
new = _book_files(books) - before
|
||||
if new:
|
||||
return new
|
||||
time.sleep(2)
|
||||
return set()
|
||||
|
||||
|
||||
def _track_id(release: dict) -> str:
|
||||
return str(release.get("source_id") or release.get("id") or release.get("guid") or "")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 1. Real Chrome solves Cloudflare end-to-end (the "spin a chrome browser" path)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_real_chrome_solves_cloudflare_end_to_end(client) -> None:
|
||||
"""Download an AA book whose slow-download is behind Cloudflare; the internal
|
||||
headless Chrome must solve the gate for the file to arrive in /books."""
|
||||
books = _books_dir()
|
||||
if books is None or not books.exists():
|
||||
pytest.skip("E2E_BOOKS_DIR not visible to the test runner")
|
||||
before = _book_files(books)
|
||||
|
||||
resp = client.direct_search(BOOK)
|
||||
assert resp.status_code == 200, f"AA search failed: {resp.status_code} {resp.text[:300]}"
|
||||
releases = client.releases_from(resp)
|
||||
assert releases, "AA search returned no releases for Moby Dick"
|
||||
|
||||
queued = client.queue_download(releases[0])
|
||||
assert queued.status_code in (200, 201, 202), (
|
||||
f"queue refused the AA release: {queued.status_code} {queued.text[:300]}"
|
||||
)
|
||||
book_id = _track_id(releases[0])
|
||||
assert book_id, f"release missing a trackable id: {releases[0]!r}"
|
||||
|
||||
state, info = client.wait_for_terminal(book_id)
|
||||
assert state in {"complete", "done", "available"}, (
|
||||
f"AA download via the Chrome-solved Cloudflare gate did not complete: "
|
||||
f"state={state} info={info!r}"
|
||||
)
|
||||
|
||||
new_files = _wait_for_new_book(books, before)
|
||||
assert new_files, (
|
||||
"download reported complete but no file landed in /books — the Cloudflare "
|
||||
"gate in front of the slow-download was not solved by Chrome"
|
||||
)
|
||||
assert all(Path(n).suffix for n in new_files), f"file written without extension: {new_files}"
|
||||
|
||||
# Secondary, explicit signal that the internal bypasser (Chrome) was engaged.
|
||||
_assert_bypasser_engaged()
|
||||
|
||||
|
||||
def _assert_bypasser_engaged() -> None:
|
||||
"""Confirm shelfmark actually routed through the internal Chrome bypasser.
|
||||
|
||||
Best-effort: reads the live shelfmark container logs. The file landing in /books
|
||||
is already proof (the slow-download was CF-gated), but this pins the mechanism.
|
||||
"""
|
||||
if shutil.which("docker") is None:
|
||||
return
|
||||
result = subprocess.run(
|
||||
["docker", "logs", "e2e-shelfmark"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
blob = (result.stdout + result.stderr).lower()
|
||||
if not blob.strip():
|
||||
return
|
||||
assert "bypass" in blob, (
|
||||
"no evidence the internal bypasser engaged during the download — the file "
|
||||
"may have arrived via an unexpected (non-Chrome) path"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 2. DoH
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_doh_enabled_and_app_healthy(client) -> None:
|
||||
"""DoH is enabled at boot and the app stays healthy (DoH init has historically
|
||||
broken startup). ``/api/config`` doesn't expose ``USE_DOH``, so verify via the
|
||||
boot log + health."""
|
||||
assert client.get("/api/health").status_code == 200
|
||||
log = _boot_log()
|
||||
if not log:
|
||||
pytest.skip("E2E_SHELFMARK_LOG not available to assert DoH")
|
||||
assert "USE_DOH=true" in log or "'USE_DOH'" in log, "USE_DOH was not synced into config at boot"
|
||||
|
||||
|
||||
# NOTE: the real torrent-client download (Prowlarr -> qBittorrent/transmission/
|
||||
# deluge/rtorrent -> /books) lives in test_cluster_clients.py, which runs under the
|
||||
# `full` profile *and* the `client-*` profiles from one client-agnostic test.
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Clusters 5/6/7 — container/entrypoint correctness, plus pointers.
|
||||
|
||||
Cluster 6 (docker/entrypoint/PUID-PGID — 9 issues / 19 fix PRs, zero shell test
|
||||
coverage today): the app must boot healthy under the configured PUID/PGID with no
|
||||
permission errors, regardless of which config profile is active. This is a
|
||||
profile-agnostic invariant, so it runs under every profile and catches
|
||||
entrypoint/permission regressions (#801, #411, #434, #171) across the matrix.
|
||||
|
||||
Clusters 5 (download clients) and 7 (audiobook/ABB) are exercised by dedicated
|
||||
stacks/sources (docker-compose.test-clients.yml and an audiobookbay mock) — see
|
||||
README. Pointers below keep them visible in the matrix without duplicating the
|
||||
prowlarr e2e flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_health_endpoint_under_every_profile(client, active_profile) -> None:
|
||||
"""Entrypoint/permission boot must succeed under the active config."""
|
||||
resp = client.get("/api/health")
|
||||
assert resp.status_code == 200, f"[{active_profile}] not healthy: {resp.text[:200]}"
|
||||
|
||||
|
||||
def test_no_permission_errors_in_boot_logs() -> None:
|
||||
"""The runner captures shelfmark boot logs into E2E_SHELFMARK_LOG; assert no
|
||||
permission/entrypoint failure markers (regression for #171/#447/#801)."""
|
||||
log_path = os.environ.get("E2E_SHELFMARK_LOG")
|
||||
if not log_path or not os.path.exists(log_path):
|
||||
pytest.skip("E2E_SHELFMARK_LOG not provided by the runner")
|
||||
with open(log_path, encoding="utf-8", errors="ignore") as fh:
|
||||
text = fh.read().lower()
|
||||
for marker in ("permission denied", "operation not permitted", "read-only file system"):
|
||||
assert marker not in text, f"boot logs contain a permission failure: {marker!r}"
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="cluster 5: covered by docker-compose.test-clients.yml + prowlarr e2e flow"
|
||||
)
|
||||
def test_download_clients_pointer() -> None: # pragma: no cover - documentation marker
|
||||
...
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="cluster 7: needs an audiobookbay mock role (tracked in README roadmap)")
|
||||
def test_audiobook_pointer() -> None: # pragma: no cover - documentation marker
|
||||
...
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Cluster 2/3 — search relevance and Anna's Archive HTML-parse robustness.
|
||||
|
||||
The recurring root cause: AA changes its DOM and the hardcoded-index parser
|
||||
silently returns zero rows, which users experience as "All download sources
|
||||
failed" (#1028) or "book not found" (#198, #293). Evidence of brittleness:
|
||||
#878/#879/#880 (hardcoded indices/selectors), plus the repeated
|
||||
"Fix AA ... after they changed layout" PRs.
|
||||
|
||||
These run under ``baseline`` (direct connection to the fake AA) so the assertions
|
||||
isolate parsing from egress concerns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.profiles("baseline")
|
||||
|
||||
|
||||
def test_search_returns_parsed_releases(client) -> None:
|
||||
"""Happy path: the parser turns the AA results table into releases."""
|
||||
resp = client.direct_search("Mistborn")
|
||||
assert resp.status_code == 200, resp.text
|
||||
releases = client.releases_from(resp)
|
||||
assert releases, "expected at least one parsed release from the AA results table"
|
||||
titles = " ".join(str(r.get("title", "")) for r in releases)
|
||||
assert "Mistborn" in titles, f"query not reflected in parsed titles: {titles[:200]}"
|
||||
|
||||
|
||||
def test_layout_drift_fails_loudly_not_silently(client) -> None:
|
||||
"""When AA's DOM changes so no row parses, the app must surface a clear
|
||||
failure — NOT an empty 200 that reads as 'book does not exist'.
|
||||
|
||||
Regression guard for #878/#879/#880 and the layout-change PRs.
|
||||
"""
|
||||
resp = client.direct_search("Mistborn", inject="layout_drift")
|
||||
releases = client.releases_from(resp)
|
||||
# Acceptable behaviours: an explicit error status, OR a 200 with an error
|
||||
# field. NOT acceptable: 200 + empty releases with no signal.
|
||||
if resp.status_code == 200:
|
||||
body = (
|
||||
resp.json()
|
||||
if resp.headers.get("content-type", "").startswith("application/json")
|
||||
else {}
|
||||
)
|
||||
has_error_signal = bool(body.get("error")) or bool(body.get("source_errors"))
|
||||
assert not releases, "parser unexpectedly produced releases from drifted DOM"
|
||||
assert has_error_signal, (
|
||||
"layout drift produced a silent empty 200 — the app must signal that "
|
||||
"the source could not be parsed (regression for #878/#879/#880)"
|
||||
)
|
||||
else:
|
||||
assert resp.status_code >= 400, resp.status_code
|
||||
|
||||
|
||||
def test_no_files_string_alongside_real_results(client) -> None:
|
||||
"""A real results table that also contains the literal 'No files found.'
|
||||
must still yield releases (false-positive guard)."""
|
||||
resp = client.direct_search("Mistborn", inject="no_files")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert client.releases_from(resp), (
|
||||
"'No files found.' substring caused a false-positive empty result"
|
||||
)
|
||||
|
||||
|
||||
def test_genuinely_empty_results_handled_cleanly(client) -> None:
|
||||
"""A true 'No files found.' page yields zero releases without a 500."""
|
||||
resp = client.direct_search("Mistborn", inject="empty")
|
||||
assert resp.status_code in (200, 404), resp.status_code
|
||||
assert client.releases_from(resp) == []
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Config-matrix invariants.
|
||||
|
||||
These tests carry NO ``profiles`` marker for the egress check, so they run under
|
||||
every profile the runner boots. Reaching the (mock) book source must succeed
|
||||
whether egress is direct, through an HTTP/SOCKS proxy, via custom DNS, or via the
|
||||
Cloudflare bypasser. That cross-product *is* the config matrix.
|
||||
|
||||
Covers the recurring "X setting silently breaks downloads" class:
|
||||
proxy ignored (#956), DNS/ISP blocks (#1028, #108), bypasser config not adhered
|
||||
(#410, #369, #267).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Profiles where reaching the (non-CF-gated) source via search is expected to work
|
||||
# end to end. Excluded: tor (boot-correctness only), and the bypasser profiles —
|
||||
# their AA is behind a Cloudflare gate that search does NOT bypass (the bypasser is
|
||||
# download-time only), so a *search* there fails by design (see test_cluster_bypasser).
|
||||
EGRESS_PROFILES = (
|
||||
"baseline",
|
||||
"dns-manual",
|
||||
"dns-blocked",
|
||||
"proxy-http",
|
||||
"proxy-socks",
|
||||
)
|
||||
|
||||
# The proxy container whose logs prove egress actually traversed the proxy.
|
||||
PROXY_CONTAINER = {
|
||||
"proxy-http": "e2e-tinyproxy",
|
||||
"proxy-socks": "e2e-microsocks",
|
||||
}
|
||||
|
||||
|
||||
def test_health_ok(client) -> None:
|
||||
resp = client.get("/api/health")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.profiles(*EGRESS_PROFILES)
|
||||
def test_source_reachable_under_active_profile(client, active_profile) -> None:
|
||||
"""The book source must be reachable regardless of egress configuration."""
|
||||
resp = client.direct_search("Mistborn")
|
||||
assert resp.status_code == 200, (
|
||||
f"[{active_profile}] direct search failed: {resp.status_code} {resp.text[:300]}"
|
||||
)
|
||||
releases = client.releases_from(resp)
|
||||
assert releases, (
|
||||
f"[{active_profile}] expected releases from the mock source but got none — "
|
||||
f"egress configuration is silently dropping the request"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.profiles("proxy-http")
|
||||
def test_proxy_mode_synced_at_boot(client) -> None:
|
||||
"""The deployment PROXY_MODE env override is applied at boot.
|
||||
|
||||
``/api/config`` returns only a frontend-facing subset (not network/proxy
|
||||
keys), so this is verified from the boot log where ENV→config sync is recorded.
|
||||
"""
|
||||
assert client.get("/api/health").status_code == 200
|
||||
path = os.environ.get("E2E_SHELFMARK_LOG")
|
||||
if not path or not Path(path).exists():
|
||||
pytest.skip("E2E_SHELFMARK_LOG not available")
|
||||
log = Path(path).read_text(encoding="utf-8", errors="ignore")
|
||||
assert "PROXY_MODE" in log, "PROXY_MODE was not synced into network config at boot"
|
||||
|
||||
|
||||
@pytest.mark.profiles("proxy-http", "proxy-socks")
|
||||
def test_egress_actually_traverses_proxy(client, active_profile) -> None:
|
||||
"""#956 guard: a configured proxy must actually *carry* the app's egress.
|
||||
|
||||
Reachability alone is not enough — the app and the mock AA share the e2e
|
||||
network, so a regression that silently ignores the proxy config would still
|
||||
reach AA directly and pass ``test_source_reachable_under_active_profile``.
|
||||
Here we drive a search and then inspect the proxy container's logs: if the
|
||||
proxy never saw the traffic, the proxy was bypassed (regression for #956).
|
||||
"""
|
||||
if shutil.which("docker") is None:
|
||||
pytest.skip("docker CLI not available to the test host")
|
||||
container = PROXY_CONTAINER[active_profile]
|
||||
|
||||
# Generate egress that *must* go through the proxy.
|
||||
resp = client.direct_search("Mistborn")
|
||||
assert resp.status_code == 200 and client.releases_from(resp), (
|
||||
f"[{active_profile}] search failed under proxy: {resp.status_code} {resp.text[:200]}"
|
||||
)
|
||||
|
||||
logs = subprocess.run(
|
||||
["docker", "logs", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
blob = (logs.stdout + logs.stderr).lower()
|
||||
assert blob.strip(), (
|
||||
f"[{active_profile}] proxy container {container!r} produced no logs after a "
|
||||
f"search — egress did not traverse the proxy (regression for #956 proxy-ignored)"
|
||||
)
|
||||
# Strongest signal (HTTP proxy logs the destination host/request explicitly).
|
||||
if active_profile == "proxy-http":
|
||||
markers = ("mock-aa", "aa.mock.test", "connect", "request", "get ")
|
||||
assert any(m in blob for m in markers), (
|
||||
f"tinyproxy logs show no AA request — proxy may be passing traffic without "
|
||||
f"the app routing through it. logs tail: ...{blob[-300:]!r}"
|
||||
)
|
||||
@@ -790,6 +790,119 @@ class TestQBittorrentClientAddDownload:
|
||||
expected_hash=expected_hash,
|
||||
)
|
||||
|
||||
def test_add_download_discovers_hash_when_extraction_fails(self, monkeypatch):
|
||||
"""Regression for #1012: a URL add without a hash adopts the new torrent's hash.
|
||||
|
||||
qBittorrent fetches .torrent URLs itself, so the add succeeds even when
|
||||
the prefetch could not determine the info_hash; the client must recover
|
||||
the hash from the torrent that appears instead of raising.
|
||||
"""
|
||||
config_values = {
|
||||
"QBITTORRENT_URL": "http://localhost:8080",
|
||||
"QBITTORRENT_USERNAME": "admin",
|
||||
"QBITTORRENT_PASSWORD": "password",
|
||||
"QBITTORRENT_CATEGORY": "books",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.qbittorrent.config.get",
|
||||
lambda key, default="": config_values.get(key, default),
|
||||
)
|
||||
|
||||
discovered_hash = "3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0"
|
||||
existing = MockTorrent(
|
||||
hash_val="a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", name="Existing Torrent"
|
||||
)
|
||||
added = MockTorrent(hash_val=discovered_hash, name="Test Download")
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_client_instance.torrents_add.return_value = "Ok."
|
||||
|
||||
torrents_before_add = create_mock_session_response([existing])
|
||||
torrents_after_add = create_mock_session_response([existing, added])
|
||||
properties_ok = create_mock_session_response({}, status_code=200)
|
||||
|
||||
def session_get(request_url, params=None, timeout=None):
|
||||
if request_url.endswith("/torrents/properties"):
|
||||
return properties_ok
|
||||
if mock_client_instance.torrents_add.called:
|
||||
return torrents_after_add
|
||||
return torrents_before_add
|
||||
|
||||
mock_client_instance._session.get.side_effect = session_get
|
||||
mock_client_class = MagicMock(return_value=mock_client_instance)
|
||||
|
||||
with patch.dict("sys.modules", {"qbittorrentapi": MagicMock(Client=mock_client_class)}):
|
||||
import importlib
|
||||
|
||||
import shelfmark.download.clients.qbittorrent as qb_module
|
||||
|
||||
importlib.reload(qb_module)
|
||||
|
||||
with patch(
|
||||
"shelfmark.download.clients.qbittorrent.extract_torrent_info",
|
||||
autospec=True,
|
||||
) as mock_extract:
|
||||
mock_extract.return_value = TorrentInfo(
|
||||
info_hash=None,
|
||||
torrent_data=None,
|
||||
is_magnet=False,
|
||||
magnet_url=None,
|
||||
)
|
||||
|
||||
client = qb_module.QBittorrentClient()
|
||||
result = client.add_download(
|
||||
"http://tracker.example/download/book.torrent", "Test Download"
|
||||
)
|
||||
|
||||
assert result == discovered_hash
|
||||
add_kwargs = mock_client_instance.torrents_add.call_args.kwargs
|
||||
assert add_kwargs["urls"] == "http://tracker.example/download/book.torrent"
|
||||
|
||||
def test_add_download_raises_when_hash_never_discovered(self, monkeypatch):
|
||||
"""Keep failing loudly when no hash is known and no new torrent appears."""
|
||||
config_values = {
|
||||
"QBITTORRENT_URL": "http://localhost:8080",
|
||||
"QBITTORRENT_USERNAME": "admin",
|
||||
"QBITTORRENT_PASSWORD": "password",
|
||||
"QBITTORRENT_CATEGORY": "books",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.qbittorrent.config.get",
|
||||
lambda key, default="": config_values.get(key, default),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.qbittorrent.time.sleep", lambda _seconds: None
|
||||
)
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_client_instance.torrents_add.return_value = "Ok."
|
||||
mock_client_instance._session.get.return_value = create_mock_session_response([])
|
||||
mock_client_class = MagicMock(return_value=mock_client_instance)
|
||||
|
||||
with patch.dict("sys.modules", {"qbittorrentapi": MagicMock(Client=mock_client_class)}):
|
||||
import importlib
|
||||
|
||||
import shelfmark.download.clients.qbittorrent as qb_module
|
||||
|
||||
importlib.reload(qb_module)
|
||||
|
||||
with patch(
|
||||
"shelfmark.download.clients.qbittorrent.extract_torrent_info",
|
||||
autospec=True,
|
||||
) as mock_extract:
|
||||
mock_extract.return_value = TorrentInfo(
|
||||
info_hash=None,
|
||||
torrent_data=None,
|
||||
is_magnet=False,
|
||||
magnet_url=None,
|
||||
)
|
||||
|
||||
client = qb_module.QBittorrentClient()
|
||||
with pytest.raises(RuntimeError, match="Could not determine torrent hash"):
|
||||
client.add_download(
|
||||
"http://tracker.example/download/book.torrent", "Test Download"
|
||||
)
|
||||
|
||||
def test_add_download_creates_category(self, monkeypatch):
|
||||
"""Test that add_download creates category if needed."""
|
||||
config_values = {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
This focuses on integration of mapping logic into the Prowlarr handler.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
@@ -209,6 +210,8 @@ def test_remap_fails_when_mapping_exists_but_path_missing():
|
||||
"localPath": str(local_dir),
|
||||
}
|
||||
]
|
||||
if key == "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT":
|
||||
return 0
|
||||
return default
|
||||
|
||||
with (
|
||||
@@ -254,6 +257,144 @@ def test_remap_fails_when_mapping_exists_but_path_missing():
|
||||
assert not local_file.exists()
|
||||
|
||||
|
||||
def test_wait_for_completed_path_allows_delayed_remapped_file(monkeypatch, tmp_path):
|
||||
import shelfmark.download.clients.base_handler as base_handler
|
||||
|
||||
local_file = tmp_path / "local" / "downloads" / "book.epub"
|
||||
remote_path = "/remote/downloads/book.epub"
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.name = "qbittorrent"
|
||||
mock_client.get_download_path.return_value = remote_path
|
||||
|
||||
def config_get(key: str, default=""):
|
||||
if key == "PROWLARR_REMOTE_PATH_MAPPINGS":
|
||||
return [
|
||||
{
|
||||
"host": "qbittorrent",
|
||||
"remotePath": "/remote/downloads",
|
||||
"localPath": str(local_file.parent),
|
||||
}
|
||||
]
|
||||
if key == "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT":
|
||||
return 1
|
||||
return default
|
||||
|
||||
probe_calls = 0
|
||||
original_probe = base_handler._probe_completed_path
|
||||
|
||||
def delayed_probe(path: Path):
|
||||
nonlocal probe_calls
|
||||
probe_calls += 1
|
||||
if path == local_file and probe_calls == 2:
|
||||
local_file.parent.mkdir(parents=True)
|
||||
local_file.write_text("synced")
|
||||
return original_probe(path)
|
||||
|
||||
handler = ProwlarrHandler()
|
||||
recorder = ProgressRecorder()
|
||||
monkeypatch.setattr(base_handler.config, "get", config_get)
|
||||
monkeypatch.setattr(base_handler, "_probe_completed_path", delayed_probe)
|
||||
monkeypatch.setattr(handler, "_completed_path_retry_interval", lambda: 0.01)
|
||||
|
||||
resolved_path, error = handler._wait_for_completed_path(
|
||||
mock_client,
|
||||
"download_id",
|
||||
cancel_flag=Event(),
|
||||
status_callback=recorder.status_callback,
|
||||
)
|
||||
|
||||
assert resolved_path == local_file
|
||||
assert error is None
|
||||
assert ("locating", "Waiting for completed files...") in recorder.status_updates
|
||||
|
||||
|
||||
def test_wait_for_completed_path_fails_fast_for_unsafe_mapping(monkeypatch, tmp_path):
|
||||
import shelfmark.download.clients.base_handler as base_handler
|
||||
|
||||
remote_dir = tmp_path / "remote" / "downloads"
|
||||
local_dir = tmp_path / "local" / "downloads"
|
||||
raw_path = f"{remote_dir}/../escape/book.epub"
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.name = "qbittorrent"
|
||||
mock_client.get_download_path.return_value = raw_path
|
||||
|
||||
def config_get(key: str, default=""):
|
||||
if key == "PROWLARR_REMOTE_PATH_MAPPINGS":
|
||||
return [
|
||||
{
|
||||
"host": "qbittorrent",
|
||||
"remotePath": str(remote_dir),
|
||||
"localPath": str(local_dir),
|
||||
}
|
||||
]
|
||||
if key == "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT":
|
||||
return 900
|
||||
return default
|
||||
|
||||
handler = ProwlarrHandler()
|
||||
recorder = ProgressRecorder()
|
||||
monkeypatch.setattr(base_handler.config, "get", config_get)
|
||||
monkeypatch.setattr(handler, "_completed_path_retry_interval", lambda: 999)
|
||||
|
||||
resolved_path, error = handler._wait_for_completed_path(
|
||||
mock_client,
|
||||
"download_id",
|
||||
cancel_flag=Event(),
|
||||
status_callback=recorder.status_callback,
|
||||
)
|
||||
|
||||
assert resolved_path is None
|
||||
assert error is not None
|
||||
assert "rejected unsafe path" in error
|
||||
assert mock_client.get_download_path.call_count == 1
|
||||
assert recorder.status_updates == []
|
||||
|
||||
|
||||
def test_wait_for_completed_path_fails_fast_for_permission_error(monkeypatch):
|
||||
import shelfmark.download.clients.base_handler as base_handler
|
||||
|
||||
raw_path = "/downloads/book.epub"
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.name = "qbittorrent"
|
||||
mock_client.get_download_path.return_value = raw_path
|
||||
|
||||
def config_get(key: str, default=""):
|
||||
if key == "PROWLARR_REMOTE_PATH_MAPPINGS":
|
||||
return []
|
||||
if key == "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT":
|
||||
return 900
|
||||
return default
|
||||
|
||||
probe_calls = 0
|
||||
|
||||
def permission_denied_probe(_path: Path):
|
||||
nonlocal probe_calls
|
||||
probe_calls += 1
|
||||
return False, PermissionError(errno.EACCES, "Permission denied", raw_path)
|
||||
|
||||
handler = ProwlarrHandler()
|
||||
recorder = ProgressRecorder()
|
||||
monkeypatch.setattr(base_handler.config, "get", config_get)
|
||||
monkeypatch.setattr(base_handler, "_probe_completed_path", permission_denied_probe)
|
||||
monkeypatch.setattr(handler, "_completed_path_retry_interval", lambda: 999)
|
||||
|
||||
resolved_path, error = handler._wait_for_completed_path(
|
||||
mock_client,
|
||||
"download_id",
|
||||
cancel_flag=Event(),
|
||||
status_callback=recorder.status_callback,
|
||||
)
|
||||
|
||||
assert resolved_path is None
|
||||
assert error is not None
|
||||
assert "not accessible" in error
|
||||
assert probe_calls == 1
|
||||
assert recorder.status_updates == []
|
||||
|
||||
|
||||
def test_remaps_windows_path_to_linux():
|
||||
"""Test that Windows paths from external download clients are correctly remapped."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
|
||||
@@ -348,6 +348,111 @@ class TestRTorrentClientAddDownload:
|
||||
|
||||
assert "RPC Error" in str(excinfo.value)
|
||||
|
||||
def test_add_download_discovers_hash_when_extraction_fails(self, monkeypatch):
|
||||
"""Regression for #1012: a URL add without a hash adopts the new download's hash.
|
||||
|
||||
rTorrent fetches .torrent URLs itself, so the add succeeds even when the
|
||||
prefetch could not determine the info_hash; the client must recover the
|
||||
hash from the download that appears instead of raising.
|
||||
"""
|
||||
config_values = {
|
||||
"RTORRENT_URL": "http://localhost:8080/RPC2",
|
||||
"RTORRENT_USERNAME": "",
|
||||
"RTORRENT_PASSWORD": "",
|
||||
"RTORRENT_DOWNLOAD_DIR": "/downloads",
|
||||
"RTORRENT_LABEL": "cwabd",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.rtorrent.config.get",
|
||||
make_config_getter(config_values),
|
||||
)
|
||||
|
||||
existing_hash = "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6A1B2"
|
||||
discovered_hash = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0"
|
||||
|
||||
mock_rpc = MagicMock()
|
||||
# First multicall2 is the pre-add snapshot; the second is the discovery
|
||||
# poll after load.start, where the new download has appeared.
|
||||
mock_rpc.d.multicall2.side_effect = [
|
||||
[[existing_hash]],
|
||||
[
|
||||
[existing_hash, "Existing Torrent", "cwabd"],
|
||||
[discovered_hash, "Test Torrent", "cwabd"],
|
||||
],
|
||||
]
|
||||
mock_xmlrpc = create_mock_xmlrpc_module()
|
||||
mock_xmlrpc.ServerProxy.return_value = mock_rpc
|
||||
|
||||
mock_torrent_info = MagicMock()
|
||||
mock_torrent_info.torrent_data = None
|
||||
mock_torrent_info.magnet_url = None
|
||||
mock_torrent_info.info_hash = None
|
||||
mock_torrent_info.is_magnet = False
|
||||
|
||||
with patch.dict("sys.modules", {"xmlrpc.client": mock_xmlrpc}):
|
||||
with patch(
|
||||
"shelfmark.download.clients.torrent_utils.extract_torrent_info",
|
||||
return_value=mock_torrent_info,
|
||||
):
|
||||
if "shelfmark.download.clients.rtorrent" in sys.modules:
|
||||
del sys.modules["shelfmark.download.clients.rtorrent"]
|
||||
|
||||
from shelfmark.download.clients.rtorrent import (
|
||||
RTorrentClient,
|
||||
)
|
||||
|
||||
client = RTorrentClient()
|
||||
result_hash = client.add_download(
|
||||
"http://tracker.example/download/book.torrent", "Test Torrent"
|
||||
)
|
||||
|
||||
assert result_hash == discovered_hash.lower()
|
||||
mock_rpc.load.start.assert_called_once()
|
||||
|
||||
def test_add_download_raises_when_hash_never_discovered(self, monkeypatch):
|
||||
"""Keep failing loudly when no hash is known and no new download appears."""
|
||||
config_values = {
|
||||
"RTORRENT_URL": "http://localhost:8080/RPC2",
|
||||
"RTORRENT_USERNAME": "",
|
||||
"RTORRENT_PASSWORD": "",
|
||||
"RTORRENT_DOWNLOAD_DIR": "/downloads",
|
||||
"RTORRENT_LABEL": "cwabd",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.rtorrent.config.get",
|
||||
make_config_getter(config_values),
|
||||
)
|
||||
|
||||
existing_hash = "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6A1B2"
|
||||
mock_rpc = MagicMock()
|
||||
mock_rpc.d.multicall2.return_value = [[existing_hash, "Existing Torrent", "cwabd"]]
|
||||
mock_xmlrpc = create_mock_xmlrpc_module()
|
||||
mock_xmlrpc.ServerProxy.return_value = mock_rpc
|
||||
|
||||
mock_torrent_info = MagicMock()
|
||||
mock_torrent_info.torrent_data = None
|
||||
mock_torrent_info.magnet_url = None
|
||||
mock_torrent_info.info_hash = None
|
||||
mock_torrent_info.is_magnet = False
|
||||
|
||||
with patch.dict("sys.modules", {"xmlrpc.client": mock_xmlrpc}):
|
||||
with patch(
|
||||
"shelfmark.download.clients.torrent_utils.extract_torrent_info",
|
||||
return_value=mock_torrent_info,
|
||||
):
|
||||
if "shelfmark.download.clients.rtorrent" in sys.modules:
|
||||
del sys.modules["shelfmark.download.clients.rtorrent"]
|
||||
|
||||
from shelfmark.download.clients import rtorrent as rtorrent_module
|
||||
|
||||
monkeypatch.setattr(rtorrent_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
client = rtorrent_module.RTorrentClient()
|
||||
with pytest.raises(RuntimeError, match="Could not determine torrent hash"):
|
||||
client.add_download(
|
||||
"http://tracker.example/download/book.torrent", "Test Torrent"
|
||||
)
|
||||
|
||||
|
||||
class TestRTorrentClientAudiobookLabel:
|
||||
"""Regression tests for issue #1025 — rTorrent audiobook label selection."""
|
||||
|
||||
@@ -499,15 +499,84 @@ class TestExtractTorrentInfo:
|
||||
assert result.torrent_data == torrent_data
|
||||
mock_get.assert_called_once()
|
||||
|
||||
def test_does_not_follow_trusted_torrent_url_redirect_to_untrusted_host(self, monkeypatch):
|
||||
"""Trusted HTTP prefetch does not continue through arbitrary redirects."""
|
||||
def test_follows_trusted_redirect_to_untrusted_host_without_api_key(self, monkeypatch):
|
||||
"""Regression for #1012 on v1.3.2.
|
||||
|
||||
Prowlarr's download endpoint commonly redirects to the indexer's own
|
||||
download link on another origin. The prefetch must follow that redirect
|
||||
(it is often the only way to learn the info_hash) but must not forward
|
||||
the Prowlarr API key to the untrusted origin.
|
||||
"""
|
||||
info_dict = {
|
||||
b"name": b"book.txt",
|
||||
b"length": 100,
|
||||
b"piece length": 16384,
|
||||
b"pieces": b"\x00" * 20,
|
||||
}
|
||||
torrent_data = bencode_encode({b"info": info_dict})
|
||||
expected_hash = hashlib.sha1(bencode_encode(info_dict)).hexdigest().lower()
|
||||
|
||||
config_values = {
|
||||
"PROWLARR_URL": "https://prowlarr.example",
|
||||
"PROWLARR_API_KEY": "secret",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.torrent_utils.config.get",
|
||||
lambda key, default="": config_values.get(key, default),
|
||||
)
|
||||
redirect = MagicMock(status_code=302)
|
||||
redirect.headers = {"Location": "https://tracker.example/download/book.torrent"}
|
||||
final = MagicMock(status_code=200, content=torrent_data)
|
||||
final.raise_for_status = MagicMock()
|
||||
mock_get = MagicMock(side_effect=[redirect, final])
|
||||
monkeypatch.setattr("shelfmark.download.clients.torrent_utils.requests.get", mock_get)
|
||||
|
||||
# No expected_hash supplied: the hash can only come from the prefetch.
|
||||
result = extract_torrent_info(
|
||||
"https://prowlarr.example/1/download?apikey=secret&indexer=7",
|
||||
fetch_torrent=True,
|
||||
)
|
||||
|
||||
assert result.info_hash == expected_hash
|
||||
assert result.torrent_data == torrent_data
|
||||
assert result.is_magnet is False
|
||||
assert mock_get.call_count == 2
|
||||
trusted_headers = mock_get.call_args_list[0].kwargs["headers"]
|
||||
untrusted_headers = mock_get.call_args_list[1].kwargs["headers"]
|
||||
assert trusted_headers.get("X-Api-Key") == "secret"
|
||||
assert "X-Api-Key" not in untrusted_headers
|
||||
|
||||
def test_redirect_to_magnet_link_returns_magnet_info(self, monkeypatch):
|
||||
"""A download URL redirecting to a magnet link yields the magnet's hash."""
|
||||
magnet = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=test"
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.torrent_utils.config.get",
|
||||
lambda key, default="": "https://prowlarr.example" if key == "PROWLARR_URL" else "",
|
||||
)
|
||||
response = MagicMock(status_code=302)
|
||||
response.headers = {"Location": magnet}
|
||||
mock_get = MagicMock(return_value=response)
|
||||
monkeypatch.setattr("shelfmark.download.clients.torrent_utils.requests.get", mock_get)
|
||||
|
||||
result = extract_torrent_info(
|
||||
"https://prowlarr.example/1/download?apikey=secret&indexer=7",
|
||||
fetch_torrent=True,
|
||||
)
|
||||
|
||||
assert result.is_magnet is True
|
||||
assert result.magnet_url == magnet
|
||||
assert result.info_hash == "3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0"
|
||||
mock_get.assert_called_once()
|
||||
|
||||
def test_gives_up_after_too_many_redirects(self, monkeypatch):
|
||||
"""A redirect loop falls back to the expected hash instead of spinning."""
|
||||
expected_hash = "3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0"
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.torrent_utils.config.get",
|
||||
lambda key, default="": "https://prowlarr.example" if key == "PROWLARR_URL" else "",
|
||||
)
|
||||
response = MagicMock(status_code=302)
|
||||
response.headers = {"Location": "https://attacker.example/book.torrent"}
|
||||
response.headers = {"Location": "https://tracker.example/loop"}
|
||||
mock_get = MagicMock(return_value=response)
|
||||
monkeypatch.setattr("shelfmark.download.clients.torrent_utils.requests.get", mock_get)
|
||||
|
||||
@@ -520,7 +589,8 @@ class TestExtractTorrentInfo:
|
||||
assert result.info_hash == expected_hash
|
||||
assert result.torrent_data is None
|
||||
assert result.is_magnet is False
|
||||
mock_get.assert_called_once()
|
||||
# Initial request plus the maximum of five followed redirects.
|
||||
assert mock_get.call_count == 6
|
||||
|
||||
|
||||
class TestExtractHashFromMagnet:
|
||||
|
||||
@@ -4,7 +4,7 @@ requires-python = ">=3.14"
|
||||
|
||||
[[package]]
|
||||
name = "apprise"
|
||||
version = "1.11.0"
|
||||
version = "1.12.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
@@ -15,9 +15,9 @@ dependencies = [
|
||||
{ name = "requests-oauthlib" },
|
||||
{ name = "tzdata", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a8/f8/83f4e2aaaa0342dc67f783bf84427d9ff5cfa4ecde3a52d9b587740a91ee/apprise-1.11.0.tar.gz", hash = "sha256:3b1e6f5365b302d1fae270c0c8007958e54224b9b7808acec69006ea27f5b8a2", size = 2337248, upload-time = "2026-05-29T17:52:29.198Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ff/f7/95a52cda9355f32c5db9ef77bac18e85977472d29c555fafd825cf60c309/apprise-1.12.0.tar.gz", hash = "sha256:ffe9860d99cbde05fd156c610d4bcb49d953200c4ed3cb2db001ed50722fc142", size = 2477874, upload-time = "2026-07-04T16:15:36.67Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/79/1483693160917522703708e45b4045946d86be634b8af2c2ac9f361f8fa3/apprise-1.11.0-py3-none-any.whl", hash = "sha256:6abc6d9f8dddedfe14b2918c3553146894ac6e3dad401deaf854b20b4c455d38", size = 1712975, upload-time = "2026-05-29T17:52:26.645Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/e9/acfea54abdd66a059ff82995d3703eca3a27df3dc1bdc6afaf8a30a854fe/apprise-1.12.0-py3-none-any.whl", hash = "sha256:28edabfec5a9d5dbcd9aa28bdfd8928ecb68764e40214b6e648eab6d974b5a93", size = 1797824, upload-time = "2026-07-04T16:15:34.228Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -44,14 +44,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "basedpyright"
|
||||
version = "1.39.7"
|
||||
version = "1.39.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nodejs-wheel-binaries" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/2f/e5/0d685b5808436c628ab8b9edad6810b889d11044a962bc42b128543910ea/basedpyright-1.39.7.tar.gz", hash = "sha256:688d913a19c417870c164c630ed9cdd83a8d8b484b30ab8e99f5dec4ae9604a6", size = 25503256, upload-time = "2026-06-07T11:33:27.266Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7a/4b/c1f4e211e50389304d6af32b9280e026a7133e3ad59bbdf8f7a3250f8bee/basedpyright-1.39.9.tar.gz", hash = "sha256:32cbea5fc8273e89df3db20daea56cb7286e419ccdfdc479c64759d2dc071901", size = 24412216, upload-time = "2026-06-27T02:19:49.834Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/f4/5b1e8ea279ce8f97a6bb1518c84fa25f5794022053ce10eab22ad3f0b51b/basedpyright-1.39.7-py3-none-any.whl", hash = "sha256:81266deb6044c9be98fb4555e4b7b1a521d8aee06b66e80858d183b0e3991140", size = 13182666, upload-time = "2026-06-07T11:33:24.119Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/d4/e1fa108710d0498a18c77b1e13897f31eab47c69aa8cfe2d2a4df746541e/basedpyright-1.39.9-py3-none-any.whl", hash = "sha256:6b0837b9eba972c71895167ab9b127e6afdbc17abc92312e3f8d15ca82a5611c", size = 13374276, upload-time = "2026-06-27T02:19:54.431Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -101,11 +101,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.5.20"
|
||||
version = "2026.6.17"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -360,11 +360,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "filelock"
|
||||
version = "3.29.4"
|
||||
version = "3.29.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e6/dc/be6cbe99670cd6e4ad387123647cb08e0c32975e223f82551e914c5568a6/filelock-3.29.4.tar.gz", hash = "sha256:10cdb3656fc44541cdf30652a93fb10ec6b05325620eb316bd26893e4201538a", size = 63028, upload-time = "2026-06-13T16:12:00.744Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/c8/35bdf04fb30755e2ed758f877edf3eb4a243c2463d3a258cc28b18b7a6e2/filelock-3.29.6.tar.gz", hash = "sha256:895c532ef3f4ef04972b9446a8c4e2931a5c399ff3c4be4c9369f2639b80f793", size = 70301, upload-time = "2026-07-06T23:08:08.577Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/13/37/a065dc3bd6e49423a6532c642ca7378d3f467b1ef44c2800c937af7f9739/filelock-3.29.4-py3-none-any.whl", hash = "sha256:dac1648087d5115554850d113e7dd8c83ab2d38e3435dde2d4f163847e57b767", size = 42757, upload-time = "2026-06-13T16:11:59.582Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/49/7467c2946ccd9617f7da38187071bdc45bb9a95df51f4d63d6622432ce4e/filelock-3.29.6-py3-none-any.whl", hash = "sha256:14d5f5597d2e0c4dbd774cfb6d8132da1db44da83732aab679d54f7dcf97ab65", size = 45478, upload-time = "2026-07-06T23:08:07.197Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -630,11 +630,11 @@ sdist = { url = "https://files.pythonhosted.org/packages/28/fa/b2ba8229b9381e8f6
|
||||
|
||||
[[package]]
|
||||
name = "mycdp"
|
||||
version = "1.3.7"
|
||||
version = "1.4.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a2/cf/8990229f1e51a1531105503254e3f76d6acd77e783ad9f9236c95a1a59d7/mycdp-1.3.7.tar.gz", hash = "sha256:0861fd85949eafd1ec794ebe58b54515c571544c983affbbc74249119ff94d21", size = 224859, upload-time = "2026-03-19T21:22:29.623Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bf/3e/8ec13aadc92724c20c470ac2c5cf9386ea785daa5681b6e3bf8043c2a7ff/mycdp-1.4.0.tar.gz", hash = "sha256:6a6c18cefbf7a5a4c17be3d00fcd2cac05497b03f995e52f05abfbfbee5ab42e", size = 246353, upload-time = "2026-06-13T05:23:28.631Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/97/0e/83918568b03bc2c7326eaf66454c1cb3d8734d34ccb1c4a21454ed9b8230/mycdp-1.3.7-py3-none-any.whl", hash = "sha256:49aa0f8cfbc608b157b3ad35999a74e32cddc674d7f0e0bbe402cdcef7ce69d6", size = 252208, upload-time = "2026-03-19T21:22:28.372Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/0a/c72f4357272634bb1db2c94e414fd5ac03304603dc9a51551955e26deee4/mycdp-1.4.0-py3-none-any.whl", hash = "sha256:e4723c4b3d52dc7962ced497beb4a201431bdfc813f8bb653376498b8f217411", size = 275242, upload-time = "2026-06-13T05:23:27.326Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -757,26 +757,26 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prek"
|
||||
version = "0.4.4"
|
||||
version = "0.4.8"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/13/3d71b3adbf385f7dc7fb6e16d6e25421fd8398b45d8f8410a328bf22bd3f/prek-0.4.4.tar.gz", hash = "sha256:4ec5771153d158a0e4473933b7fd9b51e1b1f57f2df50aeb7560ea6812226dc5", size = 470641, upload-time = "2026-06-04T07:26:07.199Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/46/e436a6eb9fdb4d3fd08d0ab7fdba19fe03a9e994ec810de57869b853bd8e/prek-0.4.8.tar.gz", hash = "sha256:d15d8bef72ab7b02c7dc01458ac9e05b3131534492b5ce9bb11c4f6f636fa868", size = 494570, upload-time = "2026-07-04T12:05:10.941Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/06/9f/68a577888edf7f2647a652b02899508ccd84e57ce1f79c51a44edfd308d7/prek-0.4.4-py3-none-linux_armv6l.whl", hash = "sha256:23cfd96a25de1c93e3c43c746643b80489e3b2fa49ca9c0ffd6022e51535c900", size = 5550271, upload-time = "2026-06-04T07:26:17.834Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/b8/5427a0023116343a8d787b446536a7fddfa5db7eec7713dd05618da2bdfe/prek-0.4.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a427b792c4436f49732b1f6ebccf221fdcc6390c148474280da9c2c6eaabc9c4", size = 5910136, upload-time = "2026-06-04T07:26:20.616Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/4f/d751e90b7e768e472e054cd41cbe502589436ca9c1a13bfe4fa9513f9cde/prek-0.4.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b998038fc92c990e03147eb5b95b0f2c394517f8857ab911aac8e092f1b9b3ab", size = 5470124, upload-time = "2026-06-04T07:26:29.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/fe/e73241c5777b6f9b6b95132febbd27f9be9e89912e9e93c0982680593af2/prek-0.4.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:9cebca8c15da4f1d6e3a25e6ae0611425c8596e926222050f2588c390e42df8a", size = 5732725, upload-time = "2026-06-04T07:26:19.133Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/a3/329bd910e7e5d9d0eb5e571f3ba48023213744e78411afb81f5ef8356cab/prek-0.4.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c8742ac26363e74c855df6215a709d5db183204d00ac0f1a722b13aed4da3cd0", size = 5457953, upload-time = "2026-06-04T07:26:23.41Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/f8/d642990513d9707398506bad45d39173d84266231f7d919899f694aefe2c/prek-0.4.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a2b7c8710546a1e894afa7ab022030cd4e21f1ee7ffb301b4360773d22f1f00f", size = 5860556, upload-time = "2026-06-04T07:26:11.692Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/17/a2e29cb278503a8c18612d8a62a15020648dc768e2e94bc4b4d4c9411e07/prek-0.4.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e02bd4d5e05c500e4d9f70f024e30d13aa361dc490724b7f476d2e35542c239f", size = 6652492, upload-time = "2026-06-04T07:26:09.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/10/ad3270b18135ee5d1af6f6cf4b0c8601b1cc2cb38d16e835081da820833a/prek-0.4.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f3a25041733de987a47e5a7bace47182a6f0e2ae5f960cb54c1d4630afd2591", size = 6113837, upload-time = "2026-06-04T07:26:24.873Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/d9/e8c201b9b41c4561673cea01c630ab604df89d13e952f87dbcb807d32588/prek-0.4.4-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0b04a0f36d07474f2a9fc5b1ba1197a1b326b2b211f39cd74cf0d4613545f7f4", size = 5729155, upload-time = "2026-06-04T07:26:26.194Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/cd/227b0494fcbc91e8fe15c2a4db9e6dfff95314ef38db3e40e6ea96db249d/prek-0.4.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f032ccbe2d6edc345f81a6d772c18cc169d63c27b5a8292bfe416b352bdfee57", size = 5590775, upload-time = "2026-06-04T07:26:22.038Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/e0/9d750b9cf21ece884afdc15668d0f005a36588fe1b0bb5ff4a8112ab51cb/prek-0.4.4-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:bad3586fcea3e913f0edf2e8f132f97889e03976b7ee3d120fd294ad4e89a5eb", size = 5437864, upload-time = "2026-06-04T07:26:30.673Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/48/e2e5c0299590ad18a253c0ac09508b615baa1b382010c511186572f711d3/prek-0.4.4-py3-none-musllinux_1_1_i686.whl", hash = "sha256:4058638532c6dcbf0076d23b9264cbdd9f0f0e320762e237a6b9e4e4b854a766", size = 5718579, upload-time = "2026-06-04T07:26:27.786Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/48/7fb3d4e7f664d1ce8ae35ec553872ffddf9fab4c5081735fdaae610b1e7c/prek-0.4.4-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:619bab14071670249777deea0cc0b29d904c4a514cf33b20e583900a544f0399", size = 6231622, upload-time = "2026-06-04T07:26:16.309Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/c0/a4ddbf38034afe67cfa97c4bd81c86429ada098e7c323218d9f9fd061566/prek-0.4.4-py3-none-win32.whl", hash = "sha256:143154b329c05b2f9fa3230e604d02d9c4297dd43f96135a8ba166772e8ecd60", size = 5240317, upload-time = "2026-06-04T07:26:08.726Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/8c/fe97b5b095187bb2f93bbe406bccf108c879e5e4c83f165809b0d16ce0fb/prek-0.4.4-py3-none-win_amd64.whl", hash = "sha256:c38c5140ae2ea55ebb02e6ca590a416664ea1af287cdd21f54daeec53a81015a", size = 5626104, upload-time = "2026-06-04T07:26:14.81Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/63/3586226d536796e65f8e725b531d6104e55caaa18659bdcb512661629586/prek-0.4.4-py3-none-win_arm64.whl", hash = "sha256:3efa28fb37b9ddbafb7759da8d497f0d36cf02a05816e15d6541f5669d5d2114", size = 5470399, upload-time = "2026-06-04T07:26:13.231Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/78/b4149c8913ced2e42debb49e261c4788a1ce431e84226921c2e1a7ea8545/prek-0.4.8-py3-none-linux_armv6l.whl", hash = "sha256:1f8f8cdc65836b571824c965daebb81b449f7e4a43894c58621f5708d5a185ed", size = 5668955, upload-time = "2026-07-04T12:04:41.588Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/5f/7f54a0087b6b2f1751aeb41266d9c15e66fd0055492814798ab818cd0414/prek-0.4.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:bce1798e96d9e3a6e6abf435da7107e81452f69edb3ca7c6f90a457355ea46e2", size = 6030947, upload-time = "2026-07-04T12:04:43.8Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/d6/f2829fc3902920c36b764a386fa303e71a8219dac25cb3827c575e84199a/prek-0.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ab3a52db17254d701c3cebb7eea58c8230aa7c1959aacfd5b5f25de18edb15d1", size = 5572593, upload-time = "2026-07-04T12:04:45.763Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/8c/c5589955bcd5e3e33b67d8bc3110818cecac82a38fd6bc8b5dfdc5de421c/prek-0.4.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:b3fcfd620523bbc3f51a21d7cd63449f659b9e2cf3582de12dd5949e23227b8f", size = 5847150, upload-time = "2026-07-04T12:04:47.419Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/9d/1f2dc91bdb79d2c4714b27eac9477a51490fba5b4731330dbbebc76bd345/prek-0.4.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42e65bc8425e9d7f1691a13ca1da2e07807d1ba76c35740833354b945131689e", size = 5573738, upload-time = "2026-07-04T12:04:49.125Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/29/69a7b58e16ecbc5f3989bf4b028018d11a82dcdd320b93d6588d72f32aa7/prek-0.4.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f578492a8e0c9bc6b4bf6dfbba8716f647d4cd0769bf10ad6cf336e3096fd392", size = 5981054, upload-time = "2026-07-04T12:04:50.842Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/cc/9b9850a60c22ed18c7755ebd2d72c6eefb37fac58149d09f6adc4691c2cf/prek-0.4.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4335f9d5beb123a3884a7fe34f57c9f0828f4fbb7666beab4298833459b104f", size = 6751350, upload-time = "2026-07-04T12:04:52.529Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/e5/c425aa7272b430630119e6757def3a2007555ba8cbeb2630e0448e7a8b7f/prek-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18a8747df9c602e052881d3efb14dd7f7d62a59bd7277ae5171c9e7661d59d84", size = 6243881, upload-time = "2026-07-04T12:04:54.703Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/da/accd3ad07fd2891d3c2777eb42435439fdf11982c51d60f087c0b6b6e102/prek-0.4.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:4db639db481d5f854eff9b3d2108889e613b8c15868bcf6bdd777c7cee577436", size = 5848846, upload-time = "2026-07-04T12:04:56.402Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/00/3477704635249f21f5f98ce444cd7690c2aa9dc8d146a045db88ef2cd8c5/prek-0.4.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:c3890a6f92316d2cf44eb50584e8d2b23a596dd70487022e61186a71a2ac0900", size = 5713942, upload-time = "2026-07-04T12:04:58.311Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/e6/3ca4fabaebeadc976d9a92d1d9130674265355ea3b728418bad61583b097/prek-0.4.8-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:fc7e15c24c591a37c6ffce5b25a021b16c299ac2649f183d812b67d665cd6551", size = 5554725, upload-time = "2026-07-04T12:04:59.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/46/2ab6aaaeff0cedb8955b2e4032071c8712382bdd423bb849718c3720180d/prek-0.4.8-py3-none-musllinux_1_1_i686.whl", hash = "sha256:36fe721704ff0c7624c1167639e23a5fe658bfd38c314f487219c9afd1eeb733", size = 5838595, upload-time = "2026-07-04T12:05:01.861Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/8b/91398f2b6cd1629d5d8ca8c85b08eca500814a374313b0193f4aaf6ab6c4/prek-0.4.8-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:162e544abc394a8124f3a4ad68efee116bad09440e679dbd1675177335c2a432", size = 6357222, upload-time = "2026-07-04T12:05:03.845Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/2a/ce5cbfaad36866134a21754640a05ecdba641fcd7ad15aa74cf3443f34f6/prek-0.4.8-py3-none-win32.whl", hash = "sha256:2602e46c8c5da7dfa69f60fcf88c2b57132ac623f49fb08bfb3094298c5f07e3", size = 5354388, upload-time = "2026-07-04T12:05:05.587Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/03/3bc908bc5f7e430315553e47dfa055f19923a3888f9afe4da19f244b5cbf/prek-0.4.8-py3-none-win_amd64.whl", hash = "sha256:7cb22da60bee41b89c4978c0bea7126a3c0ccc003dae6748cf29b53947815edc", size = 5748221, upload-time = "2026-07-04T12:05:07.559Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/a7/4295e6d5f5028171dfeb115ad38ab76bf3fe0c8df91b70d73c79aa760a94/prek-0.4.8-py3-none-win_arm64.whl", hash = "sha256:da70057f577b15d4bd121bf9dd29ee205fd4b4d75a0cafba062e84d7e8b4378b", size = 5574425, upload-time = "2026-07-04T12:05:09.595Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -901,11 +901,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyotp"
|
||||
version = "2.9.0"
|
||||
version = "2.10.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f3/b2/1d5994ba2acde054a443bd5e2d384175449c7d2b6d1a0614dbca3a63abfc/pyotp-2.9.0.tar.gz", hash = "sha256:346b6642e0dbdde3b4ff5a930b664ca82abfa116356ed48cc42c7d6590d36f63", size = 17763, upload-time = "2023-07-27T23:41:03.295Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/2d/c6/c5d96a86fd0bf6fa1bbb5c5c341ff3208638b692727a683c8289068d9a11/pyotp-2.10.0.tar.gz", hash = "sha256:d01e9703443616b03c57c700b5cbffd56a1f929c1b0f8f03131bc78c1fca9d3f", size = 18625, upload-time = "2026-06-14T03:48:49.221Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/c0/c33c8792c3e50193ef55adb95c1c3c2786fe281123291c2dbf0eaab95a6f/pyotp-2.9.0-py3-none-any.whl", hash = "sha256:81c2e5865b8ac55e825b0358e496e1d9387c811e85bb40e71a3b29b288963612", size = 13376, upload-time = "2023-07-27T23:41:01.685Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/65/33/7b83bde70eddaaaaef487751a9c3a5cc0c0be54620ded0e120ebdc401ff9/pyotp-2.10.0-py3-none-any.whl", hash = "sha256:1df2f6a1bcc3bb0716172a5215ddc2f8c7c7fd26a13df9927d52e1746934836c", size = 13768, upload-time = "2026-06-14T03:48:47.831Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -949,7 +949,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.0.3"
|
||||
version = "9.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
@@ -958,9 +958,9 @@ dependencies = [
|
||||
{ name = "pluggy" },
|
||||
{ name = "pygments" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1017,15 +1017,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pytest-rerunfailures"
|
||||
version = "16.3"
|
||||
version = "16.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "pytest" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4d/f0/74f8e685be7ecd1572c1256132f18fce3a665d7e07649a3f23b7eb2d3bec/pytest_rerunfailures-16.3.tar.gz", hash = "sha256:37c9b1231c8083e9f4e724f50f7a21241822f9516c15c700ebbf218d6452355c", size = 34148, upload-time = "2026-05-22T06:51:22.292Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/47/60/a90ca1cc6cffcb97b4260ed0ad2b7934b999d7c48abe4ea0840344862a3b/pytest_rerunfailures-16.4.tar.gz", hash = "sha256:8222d17c37eb7b9e4d6fc96a3c724ff4e1a5c97a5cc7cbb2c19e9282cfd21a11", size = 36635, upload-time = "2026-07-01T06:30:56.813Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/98/58a71d68d3126d7f6a6ed1944c37ec207a4ff3dc66cad3bed7b59d38df61/pytest_rerunfailures-16.3-py3-none-any.whl", hash = "sha256:6bdfb8ffb46c46072e6c16bdedee38b6c13eac620d9415ed5b63152cbf283170", size = 15396, upload-time = "2026-05-22T06:51:20.547Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/93/3cdcc4033444e822e01b573414b03fd37fd5533070c750477b8f5fa5224b/pytest_rerunfailures-16.4-py3-none-any.whl", hash = "sha256:f69b5beb39622c90d1e44bd945d826eff6db545dcf0b68f52b7e4ad15eaf6d6c", size = 16955, upload-time = "2026-07-01T06:30:55.333Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1055,15 +1055,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "python-socketio"
|
||||
version = "5.16.2"
|
||||
version = "5.16.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "bidict" },
|
||||
{ name = "python-engineio" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/07/dd/6fd4112b941f7d39b8171b6ba17902609bd8fa2059c3812a3c29dade13e7/python_socketio-5.16.2.tar.gz", hash = "sha256:ad88c228d921646efa436c0a0df217e364ef30ec072df4041484e54d49c15989", size = 128011, upload-time = "2026-05-21T22:03:44.418Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/32/2d/ffce71017c106b75099fea569df6518c63fee5d6202ce0cfe7b01e6f22c3/python_socketio-5.16.3.tar.gz", hash = "sha256:89b136f677ae65607a84cecda9b4d6c5377b40a97582c504c25df89af16d520e", size = 128095, upload-time = "2026-06-15T22:07:04.003Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/72/dc/0decaf5da92a7a969374474025787102d811d42aed1d32191fa338620e15/python_socketio-5.16.2-py3-none-any.whl", hash = "sha256:bef2da3374fd533aed4297f57b4f6512b52aa51604cb0da2165f401291c5ca20", size = 82137, upload-time = "2026-05-21T22:03:42.616Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0a/38/8c5e72d53ff8eb27497c4f268a7f6d9121e727a50b65248288ad79a93053/python_socketio-5.16.3-py3-none-any.whl", hash = "sha256:e7ad14202a5e6448824c7c2f86161d04e13dec05992257df5c709e6a2798c041", size = 82087, upload-time = "2026-06-15T22:07:02.498Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1127,16 +1127,16 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "qbittorrent-api"
|
||||
version = "2026.6.0"
|
||||
version = "2026.6.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "requests" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/07/cb/50d2483e5fe988b5cded54f76078ed448f021ea9f8c5ee457c1d502c644d/qbittorrent_api-2026.6.0.tar.gz", hash = "sha256:f2e25bf070960b906035917a10759042b0c288ac6adf191baf4f4848890b8213", size = 1428450, upload-time = "2026-06-06T01:15:27.12Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ed/a8/ea2ab39fdb5ef4408ac71a3329af9646c8a47d54c35a9138fa4d355f22dc/qbittorrent_api-2026.6.1.tar.gz", hash = "sha256:57d74ff2e710286973673e1268878826e9c7e5f471e632d7a8f059ef20f83fd4", size = 1428879, upload-time = "2026-06-27T00:03:19.996Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/02/3c/f94bab91cf6a4e48468c6521ca5d325b321a0e1b8df020661c009626973a/qbittorrent_api-2026.6.0-py3-none-any.whl", hash = "sha256:95b6b3dd34cfdbfbde0f745dc5f8af35491bc184af009efe139437239cdf9a30", size = 68113, upload-time = "2026-06-06T01:15:25.657Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/2f/d330c335e2ef3974ac940ee38e6325af4bea7515688496ae8d2d5b6aa49a/qbittorrent_api-2026.6.1-py3-none-any.whl", hash = "sha256:9e9e4b64383c463e716825e7b447b759c2c047d23e1399ed7a66d37f4aa9b994", size = 69432, upload-time = "2026-06-27T00:03:18.565Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1205,27 +1205,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.15.17"
|
||||
version = "0.15.20"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8c/a9/3abdf488f1bf3d24c699415e454ed554a6350d5d89ce183be1ee0a3361ac/ruff-0.15.17.tar.gz", hash = "sha256:2ec446937fd16c8c4de2674a209cc5af64d9c6f17d21fbf1151054fa0bcf5219", size = 4743346, upload-time = "2026-06-11T17:54:47.663Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566", size = 4755489, upload-time = "2026-06-25T17:20:37.578Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/db/4d/e11259f5da07cb6afb2d074c31bf09da9671993f7329d4f15d2fdc458301/ruff-0.15.17-py3-none-linux_armv6l.whl", hash = "sha256:d9feddb927fc68bd295f5eebc587a7e42cfaf9b65f60ca4a2386febff575da8f", size = 10856677, upload-time = "2026-06-11T17:54:49.533Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/3e/772d679e1a0dc058e58875bd2c0cb713a0530877b4a76fee3c7966df0d49/ruff-0.15.17-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:25805a226d741c47d274a35ad5c10a7dde175fcddfa511d7cf3da0a21eb3eab7", size = 11223443, upload-time = "2026-06-11T17:55:00.573Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/58/bd41f7688b2fd5623012605130ed70e60aa7f2244baa3d5066bdd61530c8/ruff-0.15.17-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f6ad73b14c2d18a3bf8ad7cb6974294d7f613a7898604826058e6ac64918ef4d", size = 10566458, upload-time = "2026-06-11T17:55:07.52Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/5b/733371013fcf1ec339e477ece6ab42bfe10bdd9bba8ee88a9516aa56bfc0/ruff-0.15.17-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ba0c1e4f95bcb3869d0d30cbd5917071ef2e28665abfec970cdab0492c713ed", size = 10914483, upload-time = "2026-06-11T17:55:05.501Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/cc/6f24251cc0252f7239391ccb85833f320efad14ebe5b443943f37ced6332/ruff-0.15.17-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:81647960f10bff57d2e51cadd0c3950fe598400c852863a038720ef5b8cca91e", size = 10647497, upload-time = "2026-06-11T17:54:57.733Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/dd/0d10c17ce1a1624d6fc3156309c3f834fdb5dfaad026ec90c85684f3990e/ruff-0.15.17-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e01a84ddbc8c16c23055ba3924476850f1bbc1917cebbb9376665a63e74260d", size = 11416967, upload-time = "2026-06-11T17:54:51.461Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/91/556bfb156f6144f355e831c23db00b2fc4120f86b3ce81cc5f7fd2df51f3/ruff-0.15.17-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fe9f653152f8f294f9f7e03bf3a453d8b4a27f7a59c78c8666167f2b17b96c", size = 12335770, upload-time = "2026-06-11T17:54:45.793Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/82/8b5999aa13355e926f06d9f42a32dcca862f623bf0363785ff89d607dffd/ruff-0.15.17-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c0fe88a7676e7a05b73174d4d4a59cb2ac21ff8263583f87a81a6018475a978", size = 11575441, upload-time = "2026-06-11T17:54:32.661Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/93/f10377bb04109ca0e8cbc483ff1982c54b6d418210041776f93e8cdc7fa9/ruff-0.15.17-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecfc3c7878fff94633ab0348524e093f9ce3243080416dd7d14f8ba400174719", size = 11557614, upload-time = "2026-06-11T17:54:34.698Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/a6/eeeae7f7d5493df41649ab3db92f086b2d0a30199e4efdf8e3dd7a033f24/ruff-0.15.17-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:b8461180b22420b1bdc289909410930761629fddf2a5aaf60fae1ab26cedc4c4", size = 11544450, upload-time = "2026-06-11T17:54:39.042Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/88/5991ce565129a24dd4a00db1254b3b5db2e53018cbe4018ea5a89738e727/ruff-0.15.17-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6eccbe50a038b503e7140b441aa9c7fc8c1f36edf23ebef9f4165c2f28f568b7", size = 10892524, upload-time = "2026-06-11T17:55:09.432Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/1d/0fdd248313425f55223968af04b0a42125466a8d88d21c1d99c6af0a51e8/ruff-0.15.17-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:382fc0521025f5a8ad447d8bdd523545d0d7646adb718eb1c2dac5065ec27c0f", size = 10659573, upload-time = "2026-06-11T17:54:36.824Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/0e/072e8260deb9461062ce9311ced27a8e541229a6ffd483013dd37661e43e/ruff-0.15.17-py3-none-musllinux_1_2_i686.whl", hash = "sha256:456d41fcd1b2777ad63f09a6e7121d43f7b688bbc76a800c10f7f8fb1f912c3f", size = 11127818, upload-time = "2026-06-11T17:55:03.124Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/b4/55060a34163121498014696b5f656db5b8c6963768f227dbf0d76b311073/ruff-0.15.17-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b1a04bcc94ae6194e9db05d16ad31f298a7194bfbcb08258bbe589cee1d587b8", size = 11655901, upload-time = "2026-06-11T17:54:53.562Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/71/9b29d6b87cef468d697f43c6a91e3fae4a80185779d7d5a4ef27d173439f/ruff-0.15.17-py3-none-win32.whl", hash = "sha256:596065960ab1ff593f744220c9fe6580eda00a95003cffa9f4048bb5b1bf0392", size = 10925574, upload-time = "2026-06-11T17:54:55.723Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/b2/8fc77f3723228836fa5d12497eb71c808f83782e10d058d2b15cfa14640b/ruff-0.15.17-py3-none-win_amd64.whl", hash = "sha256:6769e5fa1710b179b92e0bfa5a51735b35baea9013dadb06d5f44cbcf9547084", size = 12058788, upload-time = "2026-06-11T17:54:41.042Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/c7/c53e8dbff9c9dc4b7928773421ae294a5d28fcb8dcda1a089579d3a7e510/ruff-0.15.17-py3-none-win_arm64.whl", hash = "sha256:f3be1fbb34bcdfd146240d8fb92a709d4c2c8191348580a3c044ec60fa0b4456", size = 11355275, upload-time = "2026-06-11T17:54:43.635Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078", size = 10839665, upload-time = "2026-06-25T17:19:44.702Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b", size = 11208649, upload-time = "2026-06-25T17:19:48.787Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632", size = 10622638, upload-time = "2026-06-25T17:19:51.354Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd", size = 10984227, upload-time = "2026-06-25T17:19:54.044Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b", size = 10622882, upload-time = "2026-06-25T17:19:57.037Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267", size = 11474808, upload-time = "2026-06-25T17:20:00.357Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c", size = 12293094, upload-time = "2026-06-25T17:20:03.446Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae", size = 11526176, upload-time = "2026-06-25T17:20:06.301Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b", size = 11520767, upload-time = "2026-06-25T17:20:09.191Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487", size = 11500132, upload-time = "2026-06-25T17:20:13.602Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3", size = 10943828, upload-time = "2026-06-25T17:20:16.635Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053", size = 10645418, upload-time = "2026-06-25T17:20:19.4Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4", size = 11211770, upload-time = "2026-06-25T17:20:22.033Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460", size = 11618698, upload-time = "2026-06-25T17:20:25.259Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21", size = 10857322, upload-time = "2026-06-25T17:20:28.612Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415", size = 11993274, upload-time = "2026-06-25T17:20:31.871Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498, upload-time = "2026-06-25T17:20:35.03Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1239,7 +1239,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "selenium"
|
||||
version = "4.44.0"
|
||||
version = "4.45.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
@@ -1249,14 +1249,14 @@ dependencies = [
|
||||
{ name = "urllib3", extra = ["socks"] },
|
||||
{ name = "websocket-client" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/2d/4a/6d0a4f4a07e2a91511a51398203ee82bf6ce644a448aaa35c59b44aa9531/selenium-4.44.0.tar.gz", hash = "sha256:b03a831fcfcab9d912b4682f60718c48a04560d6c62f7496c16b7498c9a4427e", size = 993133, upload-time = "2026-05-12T22:48:19.246Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/86/48/486aa67320f27452e9f551b8608f1a59ce7091c8fe7ebc9f4eba274775d4/selenium-4.45.0.tar.gz", hash = "sha256:563f0c4102f112df1cda30d46ce6d177b2e4a7a3d4b0756902d5dc84d3a8a365", size = 1005503, upload-time = "2026-06-16T04:43:57.915Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/bc/885047e975e996cb317db31c4551caa915aafc6befea990f082c7233adc2/selenium-4.44.0-py3-none-any.whl", hash = "sha256:d01ea3e5ecad8149460a765f7cf5177194c21dcc0173093fc05427c289b1bf24", size = 9654291, upload-time = "2026-05-12T22:48:16.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/8a/6ff6beb9c7c6cc642f628df9328a8b6637f86602eff8d28e70b5d4e8bca7/selenium-4.45.0-py3-none-any.whl", hash = "sha256:1fd9d0dc08192b2f8100e264ed720f83b05d2dd3a7feff673df04e0c7580df4b", size = 9536616, upload-time = "2026-06-16T04:43:55.968Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seleniumbase"
|
||||
version = "4.49.10"
|
||||
version = "4.50.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
@@ -1320,9 +1320,9 @@ dependencies = [
|
||||
{ name = "wheel" },
|
||||
{ name = "wsproto" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/de/a7/311d233d036ca73a8672742e7e7a069173796ca38eaa92f9368bbc4ae58b/seleniumbase-4.49.10.tar.gz", hash = "sha256:f2c85c7492dc8f8747ba96f3bf847dc76379c35fc1f6088da06f7c7a07059449", size = 662965, upload-time = "2026-06-12T03:32:19.749Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1c/84/fadfc7c543e3bed55acd7350ffc2e0a9f24bae37eb792732611eb8dc1cc2/seleniumbase-4.50.5.tar.gz", hash = "sha256:6a510cfd319c879745a5314eb9b253433b414beeaeb1b7f2e9fc0d05a93f9cd1", size = 667241, upload-time = "2026-07-03T14:34:24.662Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/be/06/44c78f78a581e315b3164dbd5b8f36511b901cd025d0d4d04da76656e2e0/seleniumbase-4.49.10-py3-none-any.whl", hash = "sha256:ef72ff21a117c43c3bc8a009f2634ace404debfe74fdf79f01bae9e0541cc425", size = 668428, upload-time = "2026-06-12T03:32:16.298Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/df/ccf0aabfccbbd59141ddbe1f446056524c4b96cf5a7fa511e0859b4da580/seleniumbase-4.50.5-py3-none-any.whl", hash = "sha256:a78887c52e458402eaee07d3abc97a557f79c78c550fec4f864b8901458cc2c6", size = 672979, upload-time = "2026-07-03T14:34:21.262Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1381,7 +1381,7 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "apprise", specifier = ">=1.11.0" },
|
||||
{ name = "apprise", specifier = ">=1.12.0" },
|
||||
{ name = "authlib", specifier = ">=1.7.2,<1.8" },
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "defusedxml" },
|
||||
@@ -1401,7 +1401,7 @@ requires-dist = [
|
||||
{ name = "qbittorrent-api" },
|
||||
{ name = "rarfile" },
|
||||
{ name = "requests", extras = ["socks"] },
|
||||
{ name = "seleniumbase", marker = "extra == 'browser'", specifier = "==4.49.10" },
|
||||
{ name = "seleniumbase", marker = "extra == 'browser'", specifier = "==4.50.5" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "transmission-rpc" },
|
||||
]
|
||||
@@ -1409,12 +1409,12 @@ provides-extras = ["browser"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
{ name = "basedpyright", specifier = ">=1.39.7" },
|
||||
{ name = "basedpyright", specifier = ">=1.39.9" },
|
||||
{ name = "prek" },
|
||||
{ name = "pytest" },
|
||||
{ name = "pytest-cov" },
|
||||
{ name = "pytest-xdist", specifier = ">=3.8.0" },
|
||||
{ name = "ruff", specifier = "==0.15.17" },
|
||||
{ name = "ruff", specifier = "==0.15.20" },
|
||||
{ name = "vulture", specifier = ">=2.14" },
|
||||
]
|
||||
|
||||
@@ -1480,14 +1480,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.68.2"
|
||||
version = "4.68.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/85/05/0d5260f1f1ca784f4a4a0def9cbe6affe587f5b4025328d446c3d67765f4/tqdm-4.68.2.tar.gz", hash = "sha256:89c230e8dbc67c7615c142487111222f878c77427ea09549960f62389e258add", size = 171923, upload-time = "2026-06-09T13:26:42.539Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/87/d7/0535a28b1f5f24f6612fb3ff1e89fb1a8d160fee0f976e0aa6803862134b/tqdm-4.68.3.tar.gz", hash = "sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482", size = 170596, upload-time = "2026-06-17T07:36:52.105Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/75/1a0392bcc21c44dcdf87b3cf2d137e7829be2c083a1e38d44efca3d57a16/tqdm-4.68.2-py3-none-any.whl", hash = "sha256:d4240441fb5353290b87d6a85968c9decc131a99b8c7faa28269d829de669ede", size = 78578, upload-time = "2026-06-09T13:26:40.731Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl", hash = "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03", size = 78337, upload-time = "2026-06-17T07:36:50.132Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1536,11 +1536,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.15.0"
|
||||
version = "4.16.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+899
@@ -0,0 +1,899 @@
|
||||
#!/bin/bash
|
||||
|
||||
# WireGuard transparent egress for Shelfmark.
|
||||
#
|
||||
# Mirrors the structure of tor.sh: it is invoked from entrypoint.sh when
|
||||
# USING_WIREGUARD=true, brings up a WireGuard tunnel, installs a strict
|
||||
# kill-switch (all non-LAN egress must leave via the tunnel or be dropped),
|
||||
# and supervises the tunnel with a handshake-based healthcheck.
|
||||
#
|
||||
# Required:
|
||||
# - container started as root with cap NET_ADMIN (and NET_RAW)
|
||||
# - a WireGuard config mounted at $WIREGUARD_CONFIG (default /config/wg0.conf)
|
||||
#
|
||||
# Optional env:
|
||||
# WIREGUARD_CONFIG path to the wg-quick config (default /config/wg0.conf)
|
||||
# WIREGUARD_INTERFACE interface name (default wg0)
|
||||
# LAN_NETWORK comma-separated CIDRs kept off the tunnel so the WebUI /
|
||||
# internal download clients (Prowlarr, qBittorrent) stay
|
||||
# reachable, e.g. "172.16.0.0/12,10.0.0.0/8"
|
||||
# WIREGUARD_ENFORCE_DNS when true (default), force /etc/resolv.conf so DNS
|
||||
# lookups use a defined resolver instead of the container's
|
||||
# inherited one. The resolver used is WIREGUARD_DNS if set,
|
||||
# otherwise the tunnel config's own DNS = line.
|
||||
# WIREGUARD_DNS optional explicit resolver(s) (comma/space separated) to
|
||||
# write into /etc/resolv.conf when WIREGUARD_ENFORCE_DNS is
|
||||
# true. Use this when the VPN provider's push DNS filters
|
||||
# domains you need (e.g. Proton NetShield NXDOMAINs
|
||||
# annas-archive.org). Point it at an on-LAN encrypted
|
||||
# resolver (kept reachable via LAN_NETWORK) so queries stay
|
||||
# private while book-source domains still resolve. When this
|
||||
# is a LAN resolver, the DNS query leaves over the LAN and
|
||||
# the resolver's own upstream encryption applies; the actual
|
||||
# download still egresses through the tunnel.
|
||||
# WIREGUARD_DISABLE_IPV6 when true (default), strip IPv6 Address/AllowedIPs/DNS
|
||||
# from the config before wg-quick. Containers frequently lack
|
||||
# the ip6tables 'raw' table wg-quick needs, and IPv6 egress
|
||||
# would be an additional leak surface. Set false only if the
|
||||
# host exposes ip6tables and you explicitly want IPv6.
|
||||
# WIREGUARD_ALLOW_WEBUI_OFFTUNNEL when false (default), the kill-switch is
|
||||
# strictly fail-closed: the only off-tunnel egress is
|
||||
# loopback, the tunnel device and the LAN allowlist. Set true
|
||||
# ONLY if a NON-LAN client (e.g. a public reverse proxy on a
|
||||
# different segment) must reach the WebUI; it permits
|
||||
# app-server REPLY packets (--sport FLASK_PORT, conntrack
|
||||
# REPLY) to leave off-tunnel. This is server replies only,
|
||||
# never client-initiated egress, so it cannot leak outbound
|
||||
# browsing/downloads or the real IP for outbound requests —
|
||||
# but it is still an off-tunnel path while the tunnel is down,
|
||||
# hence opt-in. LAN WebUI clients never need it (covered by
|
||||
# LAN_NETWORK).
|
||||
|
||||
is_truthy() {
|
||||
case "${1,,}" in
|
||||
true|yes|1|y) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Validate that a token is a literal IPv4 address (four 0-255 octets). Used to
|
||||
# sanitise resolver entries before writing them to /etc/resolv.conf so a stray
|
||||
# comment/hostname/malformed token can't produce a bogus `nameserver` line that
|
||||
# silently breaks name resolution.
|
||||
is_ipv4() {
|
||||
local ip="$1" o1 o2 o3 o4
|
||||
case "$ip" in
|
||||
*[!0-9.]*) return 1 ;;
|
||||
esac
|
||||
IFS='.' read -r o1 o2 o3 o4 _extra <<< "$ip" || true
|
||||
[ -n "$o1" ] && [ -n "$o2" ] && [ -n "$o3" ] && [ -n "$o4" ] || return 1
|
||||
[ -z "${_extra:-}" ] || return 1
|
||||
for o in "$o1" "$o2" "$o3" "$o4"; do
|
||||
# Reject empty / non-numeric, and reject leading zeros (e.g. 010): glibc
|
||||
# inet_aton parses a leading-zero octet as OCTAL, so 010.0.0.1 != 10.0.0.1
|
||||
# — refuse the ambiguous form rather than silently write a resolver the
|
||||
# kernel would interpret differently. Then enforce the 0-255 range.
|
||||
case "$o" in ''|*[!0-9]*) return 1 ;; esac
|
||||
case "$o" in 0[0-9]*) return 1 ;; esac
|
||||
[ "$o" -ge 0 ] && [ "$o" -le 255 ] || return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# Validate that a token is a literal IPv6 address. Deliberately permissive (hex
|
||||
# groups and ':'), but requires at least one ':' and only hex/':' characters, so
|
||||
# it accepts real v6 resolvers while still rejecting hostnames/comments.
|
||||
is_ipv6() {
|
||||
local ip="$1"
|
||||
case "$ip" in
|
||||
*:*) : ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "$ip" in
|
||||
*[!0-9A-Fa-f:]*) return 1 ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
|
||||
|
||||
LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
|
||||
LOG_FILE="${LOG_DIR}/shelfmark_wireguard.log"
|
||||
|
||||
if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
exec 3>&1 4>&2
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
fi
|
||||
|
||||
echo "Starting WireGuard script"
|
||||
if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
echo "Log file: $LOG_FILE"
|
||||
else
|
||||
echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
|
||||
fi
|
||||
|
||||
set +x
|
||||
set -e
|
||||
|
||||
WIREGUARD_CONFIG="${WIREGUARD_CONFIG:-/config/wg0.conf}"
|
||||
WIREGUARD_INTERFACE="${WIREGUARD_INTERFACE:-wg0}"
|
||||
WIREGUARD_ENFORCE_DNS_VALUE="${WIREGUARD_ENFORCE_DNS:-true}"
|
||||
WIREGUARD_DISABLE_IPV6_VALUE="${WIREGUARD_DISABLE_IPV6:-true}"
|
||||
# Off-tunnel WebUI reachability is OPT-IN. When false (default), the kill-switch
|
||||
# is strictly fail-closed: the ONLY off-tunnel egress permitted is loopback, the
|
||||
# tunnel device, and the LAN allowlist. Set true only if you expose the WebUI to
|
||||
# a NON-LAN client (e.g. a public reverse proxy on a different segment) and
|
||||
# accept that server-reply packets on the app port may leave off-tunnel while
|
||||
# the tunnel is down. LAN clients never need this (they are covered by
|
||||
# LAN_NETWORK). See the WebUI-reply rule below.
|
||||
WIREGUARD_ALLOW_WEBUI_OFFTUNNEL_VALUE="${WIREGUARD_ALLOW_WEBUI_OFFTUNNEL:-false}"
|
||||
|
||||
echo "Build version: $BUILD_VERSION"
|
||||
echo "Release version: $RELEASE_VERSION"
|
||||
|
||||
if [ ! -f "$WIREGUARD_CONFIG" ]; then
|
||||
echo "[✗] WireGuard config not found at $WIREGUARD_CONFIG"
|
||||
echo " Mount your wg-quick config there (e.g. -v /host/wg0.conf:/config/wg0.conf:ro)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# wg-quick derives the interface name from the config file's basename, so stage
|
||||
# the config as /etc/wireguard/<interface>.conf regardless of its mounted name.
|
||||
RUNTIME_CONFIG="/etc/wireguard/${WIREGUARD_INTERFACE}.conf"
|
||||
mkdir -p /etc/wireguard
|
||||
cp "$WIREGUARD_CONFIG" "$RUNTIME_CONFIG"
|
||||
chmod 600 "$RUNTIME_CONFIG"
|
||||
|
||||
# Extract the DNS line (if any) before wg-quick, so we can enforce it ourselves.
|
||||
WG_DNS="$(grep -iE '^[[:space:]]*DNS[[:space:]]*=' "$RUNTIME_CONFIG" | head -n1 | cut -d'=' -f2- | tr ',' ' ' | xargs || true)"
|
||||
|
||||
# wg-quick will try to manage DNS via resolvconf which is not present in this
|
||||
# image; strip the DNS line and enforce it ourselves below to avoid wg-quick
|
||||
# aborting. Keep a copy for reference.
|
||||
sed -i -E '/^[[:space:]]*DNS[[:space:]]*=/d' "$RUNTIME_CONFIG"
|
||||
|
||||
# Strip IPv6 to avoid wg-quick failing on the ip6tables 'raw' table that many
|
||||
# container kernels don't expose, and to eliminate IPv6 as a leak path. This
|
||||
# removes IPv6 CIDRs from Address= and AllowedIPs= and drops all-IPv6 lines.
|
||||
if is_truthy "$WIREGUARD_DISABLE_IPV6_VALUE"; then
|
||||
echo "[*] Disabling IPv6 in tunnel config (WIREGUARD_DISABLE_IPV6=true)"
|
||||
# Remove IPv6 CIDRs (those containing a colon) from comma-separated
|
||||
# Address= and AllowedIPs= lines; drop the line entirely if nothing remains.
|
||||
awk '
|
||||
function trim(s){ sub(/^[ \t]+/,"",s); sub(/[ \t]+$/,"",s); return s }
|
||||
/^[ \t]*(Address|AllowedIPs)[ \t]*=/{
|
||||
eq=index($0,"="); key=substr($0,1,eq-1); val=substr($0,eq+1)
|
||||
n=split(val, parts, ","); out=""; sep=""
|
||||
for(i=1;i<=n;i++){ v=trim(parts[i]); if(v!="" && index(v,":")==0){ out=out sep v; sep=", " } }
|
||||
if(out==""){ next }
|
||||
print trim(key) " = " out; next
|
||||
}
|
||||
{ print }
|
||||
' "$RUNTIME_CONFIG" > "${RUNTIME_CONFIG}.v4" && mv "${RUNTIME_CONFIG}.v4" "$RUNTIME_CONFIG"
|
||||
chmod 600 "$RUNTIME_CONFIG"
|
||||
fi
|
||||
|
||||
# Keep only IPv4 nameservers from the captured DNS list when IPv6 is disabled.
|
||||
if is_truthy "$WIREGUARD_DISABLE_IPV6_VALUE" && [ -n "$WG_DNS" ]; then
|
||||
WG_DNS_V4=""
|
||||
for ns in $WG_DNS; do
|
||||
case "$ns" in
|
||||
*:*) : ;; # drop IPv6 resolver
|
||||
*) WG_DNS_V4="$WG_DNS_V4 $ns" ;;
|
||||
esac
|
||||
done
|
||||
WG_DNS="$(echo "$WG_DNS_V4" | xargs || true)"
|
||||
fi
|
||||
|
||||
echo "[*] Bringing up WireGuard interface '$WIREGUARD_INTERFACE' from $WIREGUARD_CONFIG..."
|
||||
# wg-quick unconditionally runs `sysctl -q net.ipv4.conf.all.src_valid_mark=1`,
|
||||
# but in a container /proc/sys is read-only, so that write fails even though the
|
||||
# value is already 1 (set at namespace creation via the compose `sysctls:` key /
|
||||
# docker --sysctl). Shim sysctl so that this single redundant write is a no-op
|
||||
# when the value is already correct; everything else falls through to the real
|
||||
# binary. This avoids needing --privileged or a writable /proc/sys.
|
||||
#
|
||||
# The shim is written to a PERSISTENT path (not a mktemp dir) so the supervised
|
||||
# healthcheck can reuse it when it bounces the tunnel on a stale handshake;
|
||||
# otherwise wg-quick would fail again on the same sysctl write and never recover.
|
||||
SYSCTL_SHIM_DIR="/app/wg-sysctl-shim"
|
||||
REAL_SYSCTL="$(command -v sysctl || echo /usr/sbin/sysctl)"
|
||||
mkdir -p "$SYSCTL_SHIM_DIR"
|
||||
cat > "${SYSCTL_SHIM_DIR}/sysctl" <<SHIM
|
||||
#!/bin/bash
|
||||
for arg in "\$@"; do
|
||||
case "\$arg" in
|
||||
net.ipv4.conf.all.src_valid_mark=1)
|
||||
cur="\$(cat /proc/sys/net/ipv4/conf/all/src_valid_mark 2>/dev/null)"
|
||||
if [ "\$cur" = "1" ]; then exit 0; fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
exec "${REAL_SYSCTL}" "\$@"
|
||||
SHIM
|
||||
chmod +x "${SYSCTL_SHIM_DIR}/sysctl"
|
||||
|
||||
# Helper: run wg-quick with the sysctl shim on PATH. Used for both the initial
|
||||
# bring-up and the healthcheck's recovery bounce.
|
||||
wg_quick_shimmed() {
|
||||
PATH="${SYSCTL_SHIM_DIR}:${PATH}" wg-quick "$@"
|
||||
}
|
||||
|
||||
# Capture the PRE-tunnel default route before wg-quick installs its fwmark
|
||||
# policy routing. wg-quick with AllowedIPs=0.0.0.0/0 leaves the main table's
|
||||
# default route intact but adds an `ip rule ... suppress_prefixlength 0` so the
|
||||
# main-table DEFAULT route is suppressed and traffic falls through to the wg
|
||||
# table. Crucially, suppress_prefixlength 0 only suppresses prefixlen-0 (default)
|
||||
# routes: any explicit route with prefixlen > 0 in the main table still wins.
|
||||
# A directly-connected LAN subnet therefore stays reachable (its connected
|
||||
# /NN route), but a LAN subnet on ANOTHER VLAN (e.g. a DNS resolver at
|
||||
# 10.127.222.2 when the container is only on 172.20.0.0/16) has no main-table
|
||||
# route, so it only matches the (suppressed) default and gets forced into the
|
||||
# tunnel — where a commercial VPN drops RFC1918 destinations. We record the
|
||||
# original gateway/dev here and add explicit per-CIDR LAN routes after the
|
||||
# kill-switch so off-subnet LAN (incl. the enforced resolver) stays off-tunnel.
|
||||
ORIG_DEFAULT="$(ip -4 route show default | head -n1)"
|
||||
ORIG_GW="$(printf '%s' "$ORIG_DEFAULT" | awk '{for(i=1;i<=NF;i++) if($i=="via"){print $(i+1); exit}}')"
|
||||
ORIG_DEV="$(printf '%s' "$ORIG_DEFAULT" | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')"
|
||||
[ -n "$ORIG_GW" ] && echo "[*] Pre-tunnel default gateway: $ORIG_GW dev ${ORIG_DEV:-?} (used to keep off-subnet LAN off the tunnel)"
|
||||
|
||||
# wg-quick handles: interface creation, address, route for AllowedIPs, and a
|
||||
# fwmark-based default route when AllowedIPs=0.0.0.0/0.
|
||||
wg_quick_shimmed up "$WIREGUARD_INTERFACE"
|
||||
|
||||
echo "[*] WireGuard interface state:"
|
||||
wg show "$WIREGUARD_INTERFACE" || true
|
||||
ip -o addr show "$WIREGUARD_INTERFACE" || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kill-switch (fail-closed, IPv4 + IPv6)
|
||||
# ---------------------------------------------------------------------------
|
||||
# wg-quick (with AllowedIPs=0.0.0.0/0) already installs a fwmark + suppress
|
||||
# routing that sends everything except the encrypted tunnel packets through
|
||||
# wg0, and blocks off-tunnel traffic to AllowedIPs. We add an explicit
|
||||
# filter-table kill-switch as defence in depth: default DROP on OUTPUT, allow
|
||||
# only loopback, the tunnel device, the LAN ranges, and the handshake to the
|
||||
# WireGuard endpoint(s).
|
||||
#
|
||||
# Endpoints are read from the LIVE interface (`wg show <iface> endpoints`), not
|
||||
# the config file: after wg-quick is up these are always concrete resolved
|
||||
# IP:port values, so the allow rule can never fail on a hostname (which would
|
||||
# otherwise drop the WireGuard encapsulation and break the tunnel). Each
|
||||
# endpoint is added to iptables or ip6tables depending on its address family.
|
||||
echo "[*] Installing kill-switch (iptables + ip6tables)..."
|
||||
|
||||
# ip6tables may be unusable in some container kernels (missing tables). Detect
|
||||
# once so we can fail closed on IPv6 when possible and warn otherwise.
|
||||
IP6TABLES_OK="true"
|
||||
if ! ip6tables -L OUTPUT >/dev/null 2>&1; then
|
||||
IP6TABLES_OK="false"
|
||||
echo "[!] ip6tables unavailable in this kernel; disabling IPv6 in the kernel instead so v6 egress cannot leak."
|
||||
# Belt-and-braces: if we cannot program an IPv6 kill-switch, drop IPv6
|
||||
# entirely at the stack so non-tunnel v6 egress is impossible.
|
||||
sysctl -w net.ipv6.conf.all.disable_ipv6=1 >/dev/null 2>&1 || true
|
||||
sysctl -w net.ipv6.conf.default.disable_ipv6=1 >/dev/null 2>&1 || true
|
||||
|
||||
# Verify IPv6 is actually off. If /proc/sys is read-only (common in
|
||||
# containers) the sysctl write silently no-ops and IPv6 could still leak
|
||||
# off-tunnel with no kill-switch. In that case fail closed: either the
|
||||
# operator disables IPv6 for the container (sysctls/--sysctl or the host),
|
||||
# or provides a kernel with a usable ip6tables. Allow an explicit override
|
||||
# (WIREGUARD_ALLOW_IPV6_LEAK=true) for operators who have confirmed the
|
||||
# container genuinely has no IPv6 connectivity.
|
||||
V6_DISABLED="$(cat /proc/sys/net/ipv6/conf/all/disable_ipv6 2>/dev/null || echo unknown)"
|
||||
# If the IPv6 stack is entirely absent, there is nothing to leak.
|
||||
if [ ! -e /proc/sys/net/ipv6 ]; then
|
||||
echo "[*] No IPv6 stack present in this namespace; nothing to fail closed on."
|
||||
elif [ "$V6_DISABLED" != "1" ]; then
|
||||
if is_truthy "${WIREGUARD_ALLOW_IPV6_LEAK:-false}"; then
|
||||
echo "[!] WARNING: could not disable IPv6 and ip6tables is unavailable; WIREGUARD_ALLOW_IPV6_LEAK=true set, continuing WITHOUT an IPv6 kill-switch (v6 egress may bypass the tunnel)."
|
||||
else
|
||||
echo "[✗] Cannot enforce an IPv6 kill-switch: ip6tables is unavailable AND IPv6 could not be disabled" >&2
|
||||
echo " (net.ipv6.conf.all.disable_ipv6=$V6_DISABLED; /proc/sys likely read-only)." >&2
|
||||
echo " Refusing to run with a potential IPv6 leak. Fix by either:" >&2
|
||||
echo " - disabling IPv6 for the container (e.g. compose sysctls: net.ipv6.conf.all.disable_ipv6=1," >&2
|
||||
echo " or docker run --sysctl net.ipv6.conf.all.disable_ipv6=1), or" >&2
|
||||
echo " - running on a kernel with a usable ip6tables, or" >&2
|
||||
echo " - setting WIREGUARD_ALLOW_IPV6_LEAK=true if the container has no IPv6 connectivity." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "[✓] IPv6 disabled at the kernel; no IPv6 leak path."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Allow the encrypted WireGuard handshake/data out to each peer endpoint.
|
||||
#
|
||||
# We pin the allow rule to the resolved endpoint destination IP *and* UDP port
|
||||
# (not the port alone). Allowing any UDP to that port would leave an off-tunnel
|
||||
# egress hole for arbitrary UDP to that destination port during tunnel
|
||||
# downtime/bounces (when the fwmark routes may be gone) — weakening the
|
||||
# fail-closed guarantee. Pinning the destination IP closes that hole: the only
|
||||
# off-NIC traffic this permits is the encrypted WireGuard transport to the peer
|
||||
# itself. Endpoints are read from the LIVE interface, so they are always
|
||||
# concrete resolved IPs. If the provider rotates the endpoint IP, the tunnel
|
||||
# goes stale and the healthcheck bounce re-derives the new live endpoint and
|
||||
# re-opens the corresponding IP+port rule (see refresh_endpoint_rules), so a
|
||||
# rotation self-heals on recovery without ever leaving a wildcard-port hole.
|
||||
# Rules are de-duplicated by IP+port; this function is idempotent and
|
||||
# re-runnable after a bounce.
|
||||
# Endpoint allow rules live in their OWN chain (SHELFMARK_WG_EP) that is
|
||||
# FLUSHED and repopulated from the live interface on every call. This is the
|
||||
# key difference from a plain `-A OUTPUT` approach: if the VPN provider rotates
|
||||
# the peer endpoint IP/port, the stale allow rule for the OLD endpoint would
|
||||
# otherwise persist forever (an off-tunnel UDP hole to a no-longer-used dest)
|
||||
# and the OUTPUT chain would grow unbounded across rotations. By flushing the
|
||||
# dedicated chain each sync, ONLY the current live endpoint(s) are ever
|
||||
# permitted, so the "only WireGuard transport to the current peer may leave
|
||||
# off-tunnel" guarantee holds and the ruleset stays bounded. The OUTPUT jump to
|
||||
# the chain is installed once (idempotent via -C) ahead of the trailing DROP.
|
||||
EP_CHAIN="SHELFMARK_WG_EP"
|
||||
|
||||
sync_endpoint_chain() {
|
||||
local endpoints ep ep_port ep_host ep_ip seen_v4=" " seen_v6=" " key
|
||||
|
||||
# --- IPv4 chain: ensure exists, ensure OUTPUT jumps to it, then flush. ---
|
||||
# These are FATAL on genuine failure (not just warned): if the chain can't
|
||||
# be created, the OUTPUT jump can't be installed, or the flush fails, the
|
||||
# kill-switch has no path for the encrypted WireGuard UDP transport to reach
|
||||
# the peer, so the tunnel could never handshake and we'd surface a confusing
|
||||
# generic handshake timeout later instead of the real cause. The container
|
||||
# exits fail-closed (OUTPUT default-DROP stays in force). The `-nL`/`-C`
|
||||
# guards keep the idempotent "already exists" path non-fatal.
|
||||
iptables -nL "$EP_CHAIN" >/dev/null 2>&1 || iptables -N "$EP_CHAIN" 2>/dev/null \
|
||||
|| { echo "[✗] Failed to create iptables chain $EP_CHAIN (missing NET_ADMIN / iptables error); refusing to continue." >&2; exit 1; }
|
||||
iptables -C OUTPUT -j "$EP_CHAIN" 2>/dev/null || iptables -I OUTPUT 1 -j "$EP_CHAIN" 2>/dev/null \
|
||||
|| { echo "[✗] Failed to install OUTPUT jump to $EP_CHAIN; refusing to continue (tunnel transport would be blocked)." >&2; exit 1; }
|
||||
iptables -F "$EP_CHAIN" 2>/dev/null \
|
||||
|| { echo "[✗] Failed to flush iptables chain $EP_CHAIN; refusing to continue." >&2; exit 1; }
|
||||
if [ "$IP6TABLES_OK" = "true" ]; then
|
||||
ip6tables -nL "$EP_CHAIN" >/dev/null 2>&1 || ip6tables -N "$EP_CHAIN" 2>/dev/null \
|
||||
|| { echo "[✗] Failed to create ip6tables chain $EP_CHAIN; refusing to continue." >&2; exit 1; }
|
||||
ip6tables -C OUTPUT -j "$EP_CHAIN" 2>/dev/null || ip6tables -I OUTPUT 1 -j "$EP_CHAIN" 2>/dev/null \
|
||||
|| { echo "[✗] Failed to install ip6tables OUTPUT jump to $EP_CHAIN; refusing to continue." >&2; exit 1; }
|
||||
ip6tables -F "$EP_CHAIN" 2>/dev/null \
|
||||
|| { echo "[✗] Failed to flush ip6tables chain $EP_CHAIN; refusing to continue." >&2; exit 1; }
|
||||
fi
|
||||
|
||||
# `wg show <if> endpoints` prints "<pubkey>\t<host:port>" per peer, or
|
||||
# "<pubkey>\t(none)" for a peer with no endpoint yet. Keep only tokens that
|
||||
# look like host:port (contain a colon and are not the literal "(none)").
|
||||
endpoints="$(wg show "$WIREGUARD_INTERFACE" endpoints 2>/dev/null | awk '{print $2}' | grep -F ':' | grep -v '(none)' || true)"
|
||||
for ep in $endpoints; do
|
||||
# Split host:port from the right so IPv6 colons in the host are preserved.
|
||||
ep_port="${ep##*:}"
|
||||
ep_host="${ep%:*}"
|
||||
# Require a numeric port and a non-empty host; skip anything malformed.
|
||||
case "$ep_port" in ''|*[!0-9]*) continue ;; esac
|
||||
[ -z "$ep_host" ] && continue
|
||||
if printf '%s' "$ep_host" | grep -q ':'; then
|
||||
# IPv6 endpoint -> ip6tables. Strip the surrounding [ ] brackets
|
||||
# for -d. Backslash-escape the bracket in the pattern so it is an
|
||||
# unambiguous literal '[' / ']' (not a glob character class) and
|
||||
# doesn't trip shell linters.
|
||||
ep_ip="${ep_host#\[}"; ep_ip="${ep_ip%\]}"
|
||||
key="${ep_ip}/${ep_port}"
|
||||
case "$seen_v6" in *" $key "*) continue ;; esac
|
||||
seen_v6="${seen_v6}${key} "
|
||||
if [ "$IP6TABLES_OK" = "true" ]; then
|
||||
# A failure to add the endpoint allow rule is FATAL at startup:
|
||||
# without it the kill-switch (default-DROP) blocks the encrypted
|
||||
# WireGuard transport to this peer, so the tunnel can never
|
||||
# handshake. Fail fast with a clear cause instead of surfacing a
|
||||
# generic handshake timeout later. Still fail-closed (no leak).
|
||||
ip6tables -A "$EP_CHAIN" -d "$ep_ip" -p udp --dport "$ep_port" -j ACCEPT 2>/dev/null \
|
||||
|| { echo "[✗] Failed to add IPv6 endpoint allow rule for ${ep_ip} udp/$ep_port." >&2; \
|
||||
echo " The kill-switch would block the WireGuard transport to this peer; refusing to continue." >&2; \
|
||||
exit 1; }
|
||||
else
|
||||
# IPv6 endpoint but no usable ip6tables: the IPv6 kill-switch
|
||||
# cannot permit the encrypted transport to the peer, so the
|
||||
# tunnel would never handshake. Fail fast with the real cause.
|
||||
echo "[✗] WireGuard peer endpoint is IPv6 (${ep_ip}) but ip6tables is unavailable in this kernel." >&2
|
||||
echo " The IPv6 kill-switch cannot allow the encrypted transport to the peer, so the tunnel" >&2
|
||||
echo " would never establish. Fix by running on a kernel with a usable ip6tables, or by" >&2
|
||||
echo " using an IPv4 WireGuard endpoint." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# IPv4 endpoint -> iptables.
|
||||
ep_ip="$ep_host"
|
||||
key="${ep_ip}/${ep_port}"
|
||||
case "$seen_v4" in *" $key "*) continue ;; esac
|
||||
seen_v4="${seen_v4}${key} "
|
||||
# As above: a failed add would strand the tunnel behind the kill-
|
||||
# switch. Fatal at startup, with the real cause.
|
||||
iptables -A "$EP_CHAIN" -d "$ep_ip" -p udp --dport "$ep_port" -j ACCEPT 2>/dev/null \
|
||||
|| { echo "[✗] Failed to add IPv4 endpoint allow rule for ${ep_ip} udp/$ep_port." >&2; \
|
||||
echo " The kill-switch would block the WireGuard transport to this peer; refusing to continue." >&2; \
|
||||
exit 1; }
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Back-compat alias: the startup path historically called apply_endpoint_rules.
|
||||
apply_endpoint_rules() { sync_endpoint_chain; }
|
||||
|
||||
# --- IPv4 kill-switch ---
|
||||
# Set the default policy CLOSED before touching rules, so the chain is fail-
|
||||
# closed at all times: during the (app-traffic-free) build window, and if any
|
||||
# `iptables -A` below fails under `set -e` leaving a partial chain. The trailing
|
||||
# `-j DROP` is then belt-and-braces on top of the DROP policy.
|
||||
iptables -P OUTPUT DROP
|
||||
iptables -F OUTPUT
|
||||
iptables -A OUTPUT -o lo -j ACCEPT
|
||||
iptables -A OUTPUT -o "$WIREGUARD_INTERFACE" -j ACCEPT
|
||||
# WebUI reply traffic (OPT-IN, default OFF): allow established replies FROM the
|
||||
# app's server port so a NON-LAN client (e.g. a public reverse proxy on another
|
||||
# segment) can reach the WebUI. This is an explicit exception to the strict
|
||||
# "all non-LAN egress must leave via the tunnel or be dropped" invariant: while
|
||||
# the tunnel is down these server-reply packets could egress off the physical
|
||||
# NIC. It can only ever be app-server REPLIES (scoped to --sport <FLASK_PORT> +
|
||||
# conntrack REPLY), never client-initiated egress, so it cannot leak browsing/
|
||||
# download activity or the real IP for outbound requests — but it is still an
|
||||
# off-tunnel path, so it is gated behind WIREGUARD_ALLOW_WEBUI_OFFTUNNEL. LAN
|
||||
# clients never need this (they are covered by the LAN allowlist below), so the
|
||||
# default (false) keeps the kill-switch strictly fail-closed for non-LAN egress.
|
||||
if is_truthy "$WIREGUARD_ALLOW_WEBUI_OFFTUNNEL_VALUE"; then
|
||||
iptables -A OUTPUT -p tcp --sport "${FLASK_PORT:-8084}" -m conntrack --ctstate ESTABLISHED --ctdir REPLY -j ACCEPT 2>/dev/null \
|
||||
|| echo "[!] Could not add WebUI reply allow rule (conntrack unavailable?); non-LAN WebUI clients may be unreachable"
|
||||
echo "[*] WIREGUARD_ALLOW_WEBUI_OFFTUNNEL=true: permitting off-tunnel WebUI server replies (--sport ${FLASK_PORT:-8084})."
|
||||
else
|
||||
echo "[*] Off-tunnel WebUI replies disabled (default): non-LAN egress is strictly fail-closed. Set WIREGUARD_ALLOW_WEBUI_OFFTUNNEL=true if a non-LAN reverse proxy must reach the WebUI."
|
||||
fi
|
||||
|
||||
# --- IPv6 kill-switch (fail closed) ---
|
||||
if [ "$IP6TABLES_OK" = "true" ]; then
|
||||
ip6tables -P OUTPUT DROP
|
||||
ip6tables -F OUTPUT
|
||||
ip6tables -A OUTPUT -o lo -j ACCEPT
|
||||
ip6tables -A OUTPUT -o "$WIREGUARD_INTERFACE" -j ACCEPT
|
||||
# Mirror the opt-in WebUI-reply exception on IPv6 (default OFF = fail-closed).
|
||||
if is_truthy "$WIREGUARD_ALLOW_WEBUI_OFFTUNNEL_VALUE"; then
|
||||
ip6tables -A OUTPUT -p tcp --sport "${FLASK_PORT:-8084}" -m conntrack --ctstate ESTABLISHED --ctdir REPLY -j ACCEPT 2>/dev/null \
|
||||
|| echo "[!] Could not add IPv6 WebUI reply allow rule (conntrack unavailable?); non-LAN WebUI clients may be unreachable over IPv6"
|
||||
fi
|
||||
fi
|
||||
|
||||
apply_endpoint_rules
|
||||
|
||||
# Keep LAN reachable (WebUI, Prowlarr, qBittorrent, DNS on the LAN) off-tunnel.
|
||||
DEFAULT_LAN="127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||
LAN_LIST="${LAN_NETWORK:-$DEFAULT_LAN}"
|
||||
IFS=',' read -ra LAN_CIDRS <<< "$LAN_LIST"
|
||||
for cidr in "${LAN_CIDRS[@]}"; do
|
||||
cidr="$(echo "$cidr" | xargs)"
|
||||
[ -z "$cidr" ] && continue
|
||||
if printf '%s' "$cidr" | grep -q ':'; then
|
||||
[ "$IP6TABLES_OK" = "true" ] && ip6tables -A OUTPUT -d "$cidr" -j ACCEPT 2>/dev/null
|
||||
else
|
||||
iptables -A OUTPUT -d "$cidr" -j ACCEPT
|
||||
# Firewall ACCEPT alone is not enough: a LAN subnet on another VLAN has
|
||||
# no route in the main table, so wg-quick's suppress-default rule pushes
|
||||
# it into the tunnel (where the VPN drops RFC1918). Install an explicit
|
||||
# route via the original gateway so it egresses over the LAN. Skip
|
||||
# loopback (127/8) and any directly-connected subnet (add fails -> a
|
||||
# more-specific connected route already exists and wins by longest
|
||||
# prefix, so leaving it is correct).
|
||||
case "$cidr" in
|
||||
127.*) : ;;
|
||||
*)
|
||||
if [ -n "$ORIG_GW" ] && [ -n "$ORIG_DEV" ]; then
|
||||
# Distinguish EEXIST (route already present -> benign, a more-
|
||||
# specific connected route wins by longest prefix) from a REAL
|
||||
# failure (invalid gateway/onlink, EPERM, EINVAL). Swallowing
|
||||
# all errors as "already present" would silently drop the
|
||||
# resolver route and reintroduce the DNS-dead bug with no signal.
|
||||
add_rc=0
|
||||
add_err="$(ip route add "$cidr" via "$ORIG_GW" dev "$ORIG_DEV" 2>&1)" || add_rc=$?
|
||||
if [ "$add_rc" -eq 0 ]; then
|
||||
echo "[*] LAN route added: $cidr via $ORIG_GW dev $ORIG_DEV"
|
||||
elif printf '%s' "$add_err" | grep -qiE 'exists|File exists'; then
|
||||
echo "[*] LAN route for $cidr already present (connected/explicit); leaving existing route."
|
||||
else
|
||||
echo "[!] WARNING: could not add LAN route $cidr via $ORIG_GW dev $ORIG_DEV: ${add_err}" >&2
|
||||
echo " Off-VLAN LAN (incl. the enforced resolver) in $cidr may be forced into the tunnel and dropped." >&2
|
||||
# Verify the resolver specifically still routes off-tunnel; warn loudly if it now points at the wg dev.
|
||||
if [ -n "${WIREGUARD_DNS:-}" ]; then
|
||||
for _r in $(echo "$WIREGUARD_DNS" | tr ',' ' '); do
|
||||
case "$_r" in *:*) continue ;; esac
|
||||
_rdev="$(ip -4 route get "$_r" 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')"
|
||||
if [ "$_rdev" = "$WIREGUARD_INTERFACE" ]; then
|
||||
echo "[!] WARNING: resolver $_r currently routes via the tunnel ($WIREGUARD_INTERFACE); DNS will fail (VPN drops RFC1918)." >&2
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "[!] WARNING: no pre-tunnel default gateway/dev captured (ORIG_GW='${ORIG_GW}' ORIG_DEV='${ORIG_DEV}'); cannot install an off-VLAN LAN route for $cidr. Off-subnet LAN (incl. the enforced resolver) may be unreachable." >&2
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
echo "[*] Kill-switch: LAN allowed off-tunnel -> $cidr"
|
||||
done
|
||||
|
||||
# Everything else is dropped: if the tunnel drops, non-LAN egress fails closed.
|
||||
iptables -A OUTPUT -j DROP
|
||||
if [ "$IP6TABLES_OK" = "true" ]; then
|
||||
ip6tables -A OUTPUT -j DROP
|
||||
fi
|
||||
echo "[✓] Kill-switch active (default-drop; egress only via $WIREGUARD_INTERFACE or LAN, IPv4+IPv6)."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DNS enforcement (fail-closed): pin the resolver so DNS can't silently fall
|
||||
# back to an off-tunnel path.
|
||||
# ---------------------------------------------------------------------------
|
||||
# NOTE: this deliberately does NOT force resolver traffic through the tunnel.
|
||||
# The intended model is a trusted LAN resolver kept reachable off-tunnel via
|
||||
# LAN_NETWORK (the query leaves over the LAN; the resolver encrypts its own
|
||||
# upstream, and the actual download still egresses through the tunnel). It also
|
||||
# preserves Docker's embedded resolver (127.0.0.11) when present so container-
|
||||
# name resolution keeps working, pinning the embedded resolver's upstream via
|
||||
# the container's compose `dns:` list. "Fail-closed" here means: refuse to run
|
||||
# rather than leave an inherited resolver that could leak queries off-tunnel.
|
||||
if is_truthy "$WIREGUARD_ENFORCE_DNS_VALUE"; then
|
||||
# Prefer an explicit override; fall back to the tunnel config's DNS.
|
||||
DNS_TO_USE="${WIREGUARD_DNS:-$WG_DNS}"
|
||||
# Normalise separators (commas -> spaces).
|
||||
DNS_TO_USE="$(echo "$DNS_TO_USE" | tr ',' ' ' | xargs || true)"
|
||||
|
||||
# Is DNS managed by Docker's embedded resolver? When a container is created
|
||||
# with a `dns:`/`--dns` list (or default bridge DNS), Docker writes
|
||||
# /etc/resolv.conf as a single `nameserver 127.0.0.11` and runs an embedded
|
||||
# resolver there that (a) answers container-name lookups locally (e.g.
|
||||
# prowlarr, qbit) and (b) forwards everything else to the container's
|
||||
# configured upstream resolvers. Blindly truncating this file (the previous
|
||||
# behaviour) destroys container-name resolution, so app<->Prowlarr/qBittorrent
|
||||
# by-name integration dies even though the tunnel is perfectly healthy.
|
||||
#
|
||||
# In that case we PRESERVE the embedded resolver instead of overwriting it.
|
||||
# Fail-closed DNS is still satisfied because the embedded resolver's upstream
|
||||
# is pinned by the container's `dns:` list, which MUST be a trusted resolver
|
||||
# reachable off-tunnel over the LAN (the documented LAN-resolver model at the
|
||||
# top of this script: the DNS query leaves over the LAN to a trusted resolver
|
||||
# while the actual download still egresses through the tunnel). We cannot
|
||||
# reconfigure the embedded resolver's upstream from inside the container, so
|
||||
# we leave resolv.conf as Docker wrote it and rely on `dns:` for the upstream.
|
||||
DOCKER_EMBEDDED_DNS=false
|
||||
if grep -qE '^[[:space:]]*nameserver[[:space:]]+127\.0\.0\.11([[:space:]]|$)' /etc/resolv.conf 2>/dev/null; then
|
||||
DOCKER_EMBEDDED_DNS=true
|
||||
fi
|
||||
|
||||
if is_truthy "$DOCKER_EMBEDDED_DNS"; then
|
||||
echo "[*] Docker embedded resolver (127.0.0.11) detected — preserving it so container-name"
|
||||
echo " resolution (prowlarr/qbit) keeps working. External queries are forwarded by the"
|
||||
echo " embedded resolver to the container's configured upstream (the compose 'dns:' list)."
|
||||
if [ -n "$DNS_TO_USE" ]; then
|
||||
echo "[!] WARNING: WIREGUARD_DNS='$DNS_TO_USE' cannot repoint the embedded resolver's upstream" >&2
|
||||
echo " from inside the container. External DNS is forwarded to the container's compose 'dns:'" >&2
|
||||
echo " list, NOT to WIREGUARD_DNS. If 'dns:' is unset or points at an untrusted/Proton-filtered" >&2
|
||||
echo " upstream, book-source domains may NXDOMAIN or leak. Set the container 'dns:' to that same" >&2
|
||||
echo " trusted off-tunnel LAN resolver ($DNS_TO_USE)." >&2
|
||||
else
|
||||
echo "[!] NOTE: no WIREGUARD_DNS set; the embedded resolver forwards to the container's"
|
||||
echo " configured upstream. Ensure the container's 'dns:' is a trusted LAN resolver."
|
||||
fi
|
||||
# Leave /etc/resolv.conf untouched (127.0.0.11 stays primary).
|
||||
elif [ -n "$DNS_TO_USE" ]; then
|
||||
echo "[*] Enforcing resolver(s): $DNS_TO_USE"
|
||||
# Writing /etc/resolv.conf can fail if it is a read-only bind mount.
|
||||
# If we cannot pin the resolver, the container would fall back to its
|
||||
# inherited resolver, which the LAN allowlist permits and which can leak
|
||||
# queries off-tunnel. Fail closed.
|
||||
if ! { : > /etc/resolv.conf; } 2>/dev/null; then
|
||||
echo "[✗] Could not write /etc/resolv.conf (read-only mount?)." >&2
|
||||
echo " Cannot pin the resolver, so DNS could leak off-tunnel via the inherited resolver." >&2
|
||||
echo " Provide a writable /etc/resolv.conf, or set WIREGUARD_ENFORCE_DNS=false only if" >&2
|
||||
echo " you have pinned the resolver another way." >&2
|
||||
exit 1
|
||||
fi
|
||||
# Validate each resolver token as a literal IP before writing it, so a
|
||||
# stray comment/hostname/malformed token in WIREGUARD_DNS or the config
|
||||
# DNS= line can't produce a bogus `nameserver` line (e.g. "nameserver #")
|
||||
# that silently breaks resolution. Reject IPv6 resolvers when IPv6 is
|
||||
# disabled (they'd be unusable). Fail closed if, after validation, no
|
||||
# usable resolver remains — the whole point of enforcement is to avoid
|
||||
# falling back to a leak-prone inherited resolver.
|
||||
_valid_ns=""
|
||||
for ns in $DNS_TO_USE; do
|
||||
if is_ipv4 "$ns"; then
|
||||
_valid_ns="$_valid_ns $ns"
|
||||
elif is_ipv6 "$ns"; then
|
||||
if is_truthy "$WIREGUARD_DISABLE_IPV6_VALUE"; then
|
||||
echo "[!] Ignoring IPv6 resolver '$ns' because WIREGUARD_DISABLE_IPV6=true." >&2
|
||||
else
|
||||
_valid_ns="$_valid_ns $ns"
|
||||
fi
|
||||
else
|
||||
echo "[!] Ignoring invalid resolver token '$ns' (not a literal IP address)." >&2
|
||||
fi
|
||||
done
|
||||
_valid_ns="$(echo "$_valid_ns" | xargs || true)"
|
||||
if [ -z "$_valid_ns" ]; then
|
||||
echo "[✗] WIREGUARD_ENFORCE_DNS=true but no VALID IP resolver remained after validation (from '$DNS_TO_USE')." >&2
|
||||
echo " Refusing to run, because an empty/invalid resolv.conf would leak DNS off-tunnel via the" >&2
|
||||
echo " inherited resolver. Set WIREGUARD_DNS to a literal resolver IP reachable via the tunnel" >&2
|
||||
echo " or an allowed LAN resolver (or set WIREGUARD_ENFORCE_DNS=false to accept the inherited one)." >&2
|
||||
exit 1
|
||||
fi
|
||||
for ns in $_valid_ns; do
|
||||
echo "nameserver $ns" >> /etc/resolv.conf
|
||||
done
|
||||
else
|
||||
# No embedded resolver and no resolver to enforce. Leaving the inherited
|
||||
# resolver in place would let DNS leak off-tunnel. Fail closed.
|
||||
echo "[✗] WIREGUARD_ENFORCE_DNS=true but no resolver is defined (set WIREGUARD_DNS, or a DNS= line in the config)." >&2
|
||||
echo " Refusing to run, because the inherited resolver could leak DNS off-tunnel." >&2
|
||||
echo " Either set WIREGUARD_DNS to a resolver reachable via the tunnel (or an allowed LAN" >&2
|
||||
echo " resolver), or explicitly set WIREGUARD_ENFORCE_DNS=false to accept the inherited resolver." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "[*] Leaving /etc/resolv.conf unchanged (WIREGUARD_ENFORCE_DNS=$WIREGUARD_ENFORCE_DNS_VALUE)"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Supervisor: keep the tunnel healthy and fail-closed on drop.
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[*] Configuring Supervisor..."
|
||||
mkdir -p /var/log/supervisor
|
||||
cat <<EOF > /etc/supervisor/supervisord.conf
|
||||
[supervisord]
|
||||
nodaemon=false
|
||||
logfile=/var/log/supervisor/supervisord.log
|
||||
pidfile=/var/run/supervisord.pid
|
||||
user=root
|
||||
|
||||
[unix_http_server]
|
||||
file=/var/run/supervisor.sock
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
[supervisorctl]
|
||||
serverurl=unix:///var/run/supervisor.sock
|
||||
|
||||
[program:wireguard-healthcheck]
|
||||
command=/app/wireguard_healthcheck.sh
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/var/log/supervisor/wireguard-healthcheck.log
|
||||
stderr_logfile=/var/log/supervisor/wireguard-healthcheck.err.log
|
||||
EOF
|
||||
|
||||
cat <<'HC' > /app/wireguard_healthcheck.sh
|
||||
#!/bin/bash
|
||||
# Monitors the WireGuard tunnel via handshake age. If the tunnel is stale,
|
||||
# bounce the interface. The iptables kill-switch means non-LAN egress stays
|
||||
# blocked while the tunnel is down, so this is recovery, not leak-prevention.
|
||||
|
||||
WIREGUARD_INTERFACE="${WIREGUARD_INTERFACE:-wg0}"
|
||||
# Max seconds since last handshake before we consider the tunnel stale.
|
||||
# WireGuard rehandshakes roughly every 2 minutes when there is traffic.
|
||||
STALE_AFTER="${WIREGUARD_STALE_AFTER:-180}"
|
||||
|
||||
# Reuse the persistent sysctl shim the main script wrote. The recovery bounce
|
||||
# runs `wg-quick up`, which unconditionally writes
|
||||
# net.ipv4.conf.all.src_valid_mark=1; in a container /proc/sys is read-only so
|
||||
# that write fails and wg-quick would abort, never recovering. Putting the shim
|
||||
# ahead on PATH makes that one redundant write a no-op, mirroring the initial
|
||||
# bring-up.
|
||||
SYSCTL_SHIM_DIR="/app/wg-sysctl-shim"
|
||||
if [ -x "${SYSCTL_SHIM_DIR}/sysctl" ]; then
|
||||
PATH="${SYSCTL_SHIM_DIR}:${PATH}"
|
||||
fi
|
||||
|
||||
# Detect ip6tables usability independently (this script runs under supervisor in
|
||||
# its own environment and does not inherit the parent's IP6TABLES_OK).
|
||||
if ip6tables -L OUTPUT >/dev/null 2>&1; then
|
||||
IP6TABLES_OK="true"
|
||||
else
|
||||
IP6TABLES_OK="false"
|
||||
fi
|
||||
|
||||
latest_handshake_epoch() {
|
||||
wg show "$WIREGUARD_INTERFACE" latest-handshakes 2>/dev/null \
|
||||
| awk '{print $2}' | sort -nr | head -n1
|
||||
}
|
||||
|
||||
# Force a handshake by giving the kernel a packet to send over the tunnel. On an
|
||||
# IDLE tunnel without PersistentKeepalive, WireGuard does not rehandshake, so
|
||||
# latest-handshake legitimately ages out even though the tunnel is healthy.
|
||||
# Probing before judging staleness avoids bouncing a healthy-but-idle tunnel.
|
||||
# Best-effort and fully silenced: ping bound to the wg dev (encrypted into the
|
||||
# tunnel or dropped by the kill-switch — never leaks), curl fallback to a bare
|
||||
# IP so it needs no DNS; both cannot abort the loop.
|
||||
hc_handshake_probe() {
|
||||
ping -c 1 -W 1 -I "$WIREGUARD_INTERFACE" 1.1.1.1 >/dev/null 2>&1 \
|
||||
|| curl -s --max-time 3 -o /dev/null "https://1.1.1.1" >/dev/null 2>&1 \
|
||||
|| true
|
||||
}
|
||||
|
||||
# Re-open the WireGuard endpoint(s) in the kill-switch from the LIVE interface.
|
||||
# The endpoint allow rules are first derived at startup, but a provider IP
|
||||
# rotation or NAT rebinding can change the peer endpoint later. We pin each rule
|
||||
# to the resolved endpoint destination IP *and* UDP port (not the port alone):
|
||||
# a wildcard-port rule would leave an off-tunnel UDP hole to that port during/
|
||||
# after a bounce while the tunnel isn't fully up.
|
||||
#
|
||||
# Endpoint rules live in a DEDICATED chain (SHELFMARK_WG_EP) that we FLUSH and
|
||||
# repopulate from the live interface every cycle. This is deliberately not an
|
||||
# additive `-I OUTPUT` scheme: appending/inserting-only would leave the OLD
|
||||
# endpoint's allow rule in place forever after a provider rotation (a permanent
|
||||
# off-tunnel UDP hole to a dest we no longer use) and grow the chain unbounded.
|
||||
# Flushing first means only the CURRENT live endpoint(s) are ever permitted, so
|
||||
# a rotated endpoint IP is re-permitted on recovery while the stale one is
|
||||
# removed in the same pass, and everything else stays forced through the tunnel
|
||||
# by the default-DROP. IPv6 hosts have their [] brackets stripped for -d; IPv6
|
||||
# rules only run when ip6tables is usable. Must mirror sync_endpoint_chain in the
|
||||
# parent script (this runs in the supervised healthcheck's own process).
|
||||
EP_CHAIN="${EP_CHAIN:-SHELFMARK_WG_EP}"
|
||||
refresh_endpoint_rules() {
|
||||
local eps ep ep_host ep_port ep_ip seen_v4=" " seen_v6=" " key
|
||||
# Ensure the chain exists, OUTPUT jumps to it, then flush stale entries.
|
||||
iptables -nL "$EP_CHAIN" >/dev/null 2>&1 || iptables -N "$EP_CHAIN" 2>/dev/null || true
|
||||
iptables -C OUTPUT -j "$EP_CHAIN" 2>/dev/null || iptables -I OUTPUT 1 -j "$EP_CHAIN" 2>/dev/null || true
|
||||
iptables -F "$EP_CHAIN" 2>/dev/null || true
|
||||
if [ "$IP6TABLES_OK" = "true" ]; then
|
||||
ip6tables -nL "$EP_CHAIN" >/dev/null 2>&1 || ip6tables -N "$EP_CHAIN" 2>/dev/null || true
|
||||
ip6tables -C OUTPUT -j "$EP_CHAIN" 2>/dev/null || ip6tables -I OUTPUT 1 -j "$EP_CHAIN" 2>/dev/null || true
|
||||
ip6tables -F "$EP_CHAIN" 2>/dev/null || true
|
||||
fi
|
||||
# Keep only host:port tokens; skip "(none)" and malformed entries.
|
||||
eps="$(wg show "$WIREGUARD_INTERFACE" endpoints 2>/dev/null | awk '{print $2}' | grep -F ':' | grep -v '(none)' || true)"
|
||||
for ep in $eps; do
|
||||
ep_port="${ep##*:}"
|
||||
ep_host="${ep%:*}"
|
||||
case "$ep_port" in ''|*[!0-9]*) continue ;; esac
|
||||
[ -z "$ep_host" ] && continue
|
||||
if printf '%s' "$ep_host" | grep -q ':'; then
|
||||
# Strip the surrounding [ ] brackets from an IPv6 endpoint for -d;
|
||||
# backslash-escape the bracket so the pattern is an unambiguous
|
||||
# literal, not a glob character class (mirrors sync_endpoint_chain).
|
||||
ep_ip="${ep_host#\[}"; ep_ip="${ep_ip%\]}"
|
||||
key="${ep_ip}/${ep_port}"
|
||||
case "$seen_v6" in *" $key "*) continue ;; esac
|
||||
seen_v6="${seen_v6}${key} "
|
||||
[ "$IP6TABLES_OK" = "true" ] && ip6tables -A "$EP_CHAIN" -d "$ep_ip" -p udp --dport "$ep_port" -j ACCEPT 2>/dev/null || true
|
||||
else
|
||||
ep_ip="$ep_host"
|
||||
key="${ep_ip}/${ep_port}"
|
||||
case "$seen_v4" in *" $key "*) continue ;; esac
|
||||
seen_v4="${seen_v4}${key} "
|
||||
iptables -A "$EP_CHAIN" -d "$ep_ip" -p udp --dport "$ep_port" -j ACCEPT 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
FAIL_COUNT=0
|
||||
# Give the first handshake time to complete before judging health.
|
||||
sleep 20
|
||||
|
||||
while true; do
|
||||
# Proactively re-open the current live endpoint IP+port every cycle. If the
|
||||
# provider rotates the endpoint IP while the tunnel is up, this adds the new
|
||||
# allow rule before the DROP so the next handshake to the new endpoint is
|
||||
# not blocked, minimising recovery delay (rather than waiting for a bounce).
|
||||
refresh_endpoint_rules
|
||||
|
||||
HS="$(latest_handshake_epoch)"
|
||||
NOW="$(date +%s)"
|
||||
|
||||
if [ -z "$HS" ] || [ "$HS" = "0" ]; then
|
||||
AGE=99999
|
||||
else
|
||||
AGE=$((NOW - HS))
|
||||
fi
|
||||
|
||||
if [ "$AGE" -le "$STALE_AFTER" ]; then
|
||||
FAIL_COUNT=0
|
||||
else
|
||||
# Stale handshake — but on an idle tunnel without PersistentKeepalive the
|
||||
# handshake ages out legitimately. Force a handshake with a probe and
|
||||
# re-evaluate AGE before counting this as a failure, so a healthy-but-
|
||||
# idle tunnel is not bounced every few minutes (needless churn + noisy
|
||||
# logs). A genuinely dead tunnel won't handshake, so AGE stays stale and
|
||||
# the failure still accrues -> real outages are still detected.
|
||||
hc_handshake_probe
|
||||
sleep 3
|
||||
HS="$(latest_handshake_epoch)"
|
||||
NOW="$(date +%s)"
|
||||
if [ -z "$HS" ] || [ "$HS" = "0" ]; then
|
||||
AGE=99999
|
||||
else
|
||||
AGE=$((NOW - HS))
|
||||
fi
|
||||
if [ "$AGE" -le "$STALE_AFTER" ]; then
|
||||
FAIL_COUNT=0
|
||||
else
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
# Clamp so the counter can't grow unbounded across a long outage.
|
||||
[ "$FAIL_COUNT" -gt 3 ] && FAIL_COUNT=3
|
||||
echo "$(date): WireGuard handshake stale after probe (age=${AGE}s, fail=${FAIL_COUNT})"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$FAIL_COUNT" -ge 3 ]; then
|
||||
echo "$(date): restart trigger - bouncing $WIREGUARD_INTERFACE"
|
||||
wg-quick down "$WIREGUARD_INTERFACE" 2>/dev/null || true
|
||||
# Bring the tunnel back up via the sysctl shim (read-only /proc/sys).
|
||||
# The DROP policy + rules stay in place so we never leak during the bounce.
|
||||
if wg-quick up "$WIREGUARD_INTERFACE" 2>/dev/null; then
|
||||
# The peer endpoint may have rotated; re-open it so the kill-switch
|
||||
# does not strand the reconnect. Only reset the failure counter on a
|
||||
# successful bring-up.
|
||||
refresh_endpoint_rules
|
||||
FAIL_COUNT=0
|
||||
else
|
||||
# Leave FAIL_COUNT at/above the threshold so the NEXT cycle retries
|
||||
# the bounce immediately instead of waiting for 3 more stale cycles.
|
||||
# Cap it so it can't overflow on a long outage.
|
||||
echo "$(date): wg-quick up failed; will retry next cycle (still fail-closed)"
|
||||
FAIL_COUNT=3
|
||||
fi
|
||||
sleep 15
|
||||
fi
|
||||
|
||||
sleep 30
|
||||
done
|
||||
HC
|
||||
chmod +x /app/wireguard_healthcheck.sh
|
||||
|
||||
echo "[*] Starting Supervisor..."
|
||||
/usr/bin/supervisord -c /etc/supervisor/supervisord.conf
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verify egress actually leaves via the tunnel before handing off to the app.
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[*] Waiting for first WireGuard handshake (up to 60s)..."
|
||||
HANDSHAKE_TIMEOUT=60
|
||||
HANDSHAKE_START=$(date +%s)
|
||||
# WireGuard only performs a handshake when the kernel actually has a packet to
|
||||
# send over the tunnel (or every PersistentKeepalive interval). A config without
|
||||
# PersistentKeepalive and with no app traffic yet would never handshake, so this
|
||||
# passive poll could time out and abort a perfectly healthy tunnel. Emit a tiny
|
||||
# best-effort probe toward a public address routed INTO the tunnel (AllowedIPs=
|
||||
# 0.0.0.0/0 -> fwmark default; kill-switch already permits -o wg0), which gives
|
||||
# the kernel something to send and triggers the initial handshake. The probe
|
||||
# target need not reply; the outbound attempt alone initiates the handshake.
|
||||
wg_handshake_probe() {
|
||||
# Prefer ping (cheapest, no DNS); fall back to a curl to a bare IP so we do
|
||||
# not depend on DNS being up yet. Both are best-effort and fully silenced.
|
||||
ping -c 1 -W 1 -I "$WIREGUARD_INTERFACE" 1.1.1.1 >/dev/null 2>&1 \
|
||||
|| curl -s --max-time 3 -o /dev/null "https://1.1.1.1" >/dev/null 2>&1 \
|
||||
|| true
|
||||
}
|
||||
while true; do
|
||||
wg_handshake_probe
|
||||
HS="$(wg show "$WIREGUARD_INTERFACE" latest-handshakes 2>/dev/null | awk '{print $2}' | sort -nr | head -n1)"
|
||||
if [ -n "$HS" ] && [ "$HS" != "0" ]; then
|
||||
echo "[✓] WireGuard handshake established."
|
||||
break
|
||||
fi
|
||||
if [ $(($(date +%s) - HANDSHAKE_START)) -ge $HANDSHAKE_TIMEOUT ]; then
|
||||
echo "[✗] No WireGuard handshake after ${HANDSHAKE_TIMEOUT}s. Aborting (fail closed)."
|
||||
# Tidy up before aborting so a still-running supervised healthcheck can't
|
||||
# keep bouncing / wg-quick up a tunnel the operator has decided to
|
||||
# abandon. The container exits when entrypoint sees this non-zero exit
|
||||
# (kill-switch DROP stays in force throughout), so this is hygiene, not
|
||||
# leak-prevention.
|
||||
supervisorctl -c /etc/supervisor/supervisord.conf stop wireguard-healthcheck >/dev/null 2>&1 || true
|
||||
supervisorctl -c /etc/supervisor/supervisord.conf shutdown >/dev/null 2>&1 || true
|
||||
wg-quick down "$WIREGUARD_INTERFACE" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "[*] Verifying external egress IP is the tunnel (not the host)..."
|
||||
EGRESS_IP="$(curl -s --max-time 15 https://api.ipify.org 2>/dev/null || true)"
|
||||
if [ -n "$EGRESS_IP" ]; then
|
||||
echo "[✓] External egress IP via tunnel: $EGRESS_IP"
|
||||
else
|
||||
echo "[!] Could not determine egress IP (endpoint may block ipify). Tunnel handshake is up; continuing."
|
||||
fi
|
||||
|
||||
echo "[*] End of WireGuard script"
|
||||
Reference in New Issue
Block a user