Compare commits

..
Author SHA1 Message Date
Alex 7a4c913809 Clarify Docker digest update group 2026-05-12 12:14:29 +01:00
Alex 230015ad57 Add Dependabot update cooldown 2026-05-12 12:09:27 +01:00
Alex 28bf26414d Run uv Dependabot checks daily 2026-05-12 11:30:29 +01:00
138 changed files with 1120 additions and 7996 deletions
-8
View File
@@ -42,15 +42,7 @@ updates:
open-pull-requests-limit: 5
groups:
docker-base-image-digests:
# Exclude python from the group on purpose. Dependabot's Docker
# pre-release filter is bypassed for *grouped* updates
# (dependabot-core#9496), so a grouped python update proposes pre-release
# tags like python:3.15.0b2 as if they were a normal stable minor bump.
# Updated individually, python is filtered correctly: alpha/beta/rc tags
# are skipped and only stable releases (e.g. 3.15.0 once final) are
# proposed. node + uv stay grouped into a single digest PR.
patterns: ["*"]
exclude-patterns: ["python"]
ignore:
# Node.js: block major-version bumps so dependabot never proposes
# moving from one LTS line to a non-LTS "Current" release (e.g. 24 -> 25).
@@ -67,10 +67,10 @@ jobs:
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Log in to the Container registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -78,13 +78,7 @@ jobs:
- name: Extract metadata for ${{ matrix.target }} image
id: meta
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
env:
# Annotate both the per-platform manifests and the multi-arch image
# index. The index level is what manifest-list consumers (Renovate's
# minimumReleaseAge soak check, provenance/SBOM tooling) read for the
# standard org.opencontainers.image.* annotations, including `created`.
DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
tags: |
@@ -96,11 +90,11 @@ jobs:
type=ref,event=tag
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build and push ${{ matrix.target }} Docker image
id: push
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
platforms: linux/amd64,linux/arm64
context: .
@@ -111,11 +105,10 @@ jobs:
RELEASE_VERSION=${{ github.ref_name }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
annotations: ${{ steps.meta.outputs.annotations }}
- name: Generate artifact attestation for ${{ matrix.target }} image
if: github.event_name != 'pull_request'
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
subject-digest: ${{ steps.push.outputs.digest }}
@@ -134,14 +127,14 @@ jobs:
LEGACY_NAME: calibre-web-automated-book-downloader
steps:
- name: Log in to registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Create legacy aliases
run: |
+12 -12
View File
@@ -13,10 +13,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install uv and Python
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.3"
python-version: "3.14"
@@ -39,10 +39,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install uv and Python
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.3"
python-version: "3.14"
@@ -59,10 +59,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install uv and Python
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.3"
python-version: "3.14"
@@ -78,13 +78,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build shelfmark-lite image
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
target: shelfmark-lite
@@ -99,7 +99,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
@@ -122,7 +122,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
@@ -142,7 +142,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+4 -4
View File
@@ -22,17 +22,17 @@ jobs:
language: [python, javascript-typescript]
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Initialize CodeQL
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
uses: github/codeql-action/autobuild@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
with:
category: "/language:${{ matrix.language }}"
-130
View File
@@ -1,130 +0,0 @@
name: E2E Platform
# Hermetic end-to-end matrix: boots the app under test against mock
# Anna's Archive / Cloudflare / bypasser / DNS / proxy / Tor / real torrent
# clients and runs the cluster suite under each config profile.
#
# On a PR that touches relevant code, this runs a fast core subset *and* the heavy
# `full` profile (real Chrome solving Cloudflare + DoH + real qBittorrent). The
# `e2e-required` job aggregates them into ONE status check — make that check a
# required status check in branch protection to block merges on any e2e failure
# (see tests/e2e/platform/README.md "Gating PRs").
on:
pull_request:
schedule:
- cron: "0 4 * * *" # nightly full matrix
workflow_dispatch:
concurrency:
group: e2e-platform-${{ github.ref }}
cancel-in-progress: true
jobs:
# Detect whether anything that affects the e2e platform changed. This lets the
# required check always report (never stuck "pending") while only spending CI on
# PRs that can actually break the e2e stack.
changes:
runs-on: ubuntu-latest
outputs:
relevant: ${{ steps.filter.outputs.relevant }}
steps:
- uses: actions/checkout@v7
- uses: dorny/paths-filter@v4.0.1
id: filter
with:
filters: |
relevant:
- 'shelfmark/**'
- 'entrypoint.sh'
- 'tor.sh'
- 'Dockerfile'
- 'tests/e2e/platform/**'
- '.github/workflows/e2e-platform.yml'
select-profiles:
needs: changes
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
runs-on: ubuntu-latest
outputs:
profiles: ${{ steps.pick.outputs.profiles }}
steps:
- id: pick
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
echo 'profiles=["baseline","bypasser-external","dns-blocked"]' >> "$GITHUB_OUTPUT"
else
echo 'profiles=["baseline","bypasser-external","bypasser-disabled","dns-manual","dns-blocked","dns-doh","proxy-http","proxy-socks","tor","client-transmission","client-deluge","client-qbittorrent-delayed"]' >> "$GITHUB_OUTPUT"
fi
e2e:
needs: select-profiles
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }}
name: e2e (${{ matrix.profile }})
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install uv and Python
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
with:
python-version: "3.14"
enable-cache: true
- name: Sync dependencies
run: make install-python-dev
- name: Run e2e platform (${{ matrix.profile }})
run: tests/e2e/platform/run-e2e.sh env/${{ matrix.profile }}.env
- name: Dump shelfmark logs on failure
if: failure()
run: cat tests/e2e/platform/.state/shelfmark.${{ matrix.profile }}.log || true
# Heavy "everything real" job: real Chrome internal bypasser solving Cloudflare +
# DoH + real qBittorrent webseed download. Runs on relevant PRs and nightly.
e2e-full:
needs: changes
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
runs-on: ubuntu-latest
name: e2e (full — real Chrome + qBittorrent)
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install uv and Python
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
with:
python-version: "3.14"
enable-cache: true
- name: Sync dependencies
run: make install-python-dev
- name: Run full pipeline
run: tests/e2e/platform/run-e2e.sh env/full.env
- name: Dump logs on failure
if: failure()
run: |
cat tests/e2e/platform/.state/shelfmark.full.log || true
docker logs e2e-qbittorrent || true
# Single aggregated gate. ALWAYS runs (so a required check never hangs "pending"
# on unrelated PRs) and FAILS if any e2e job failed/was cancelled. Make THIS the
# required status check in branch protection.
e2e-required:
name: e2e required
needs: [e2e, e2e-full]
if: always()
runs-on: ubuntu-latest
steps:
- name: Aggregate e2e results
run: |
matrix='${{ needs.e2e.result }}'
full='${{ needs.e2e-full.result }}'
echo "e2e matrix=$matrix, e2e-full=$full"
# success or skipped (unrelated PR) is OK; failure/cancelled blocks.
for r in "$matrix" "$full"; do
if [ "$r" = "failure" ] || [ "$r" = "cancelled" ]; then
echo "::error::An e2e platform job did not pass — blocking."
exit 1
fi
done
echo "All e2e platform jobs passed (or were skipped as not relevant)."
-4
View File
@@ -166,10 +166,6 @@ ENV/
env.bak/
venv.bak/
# ...but the e2e platform test profiles live in an env/ dir and must be tracked
!tests/e2e/platform/env/
!tests/e2e/platform/env/*.env
# Spyder project settings
.spyderproject
.spyproject
+11 -49
View File
@@ -4,7 +4,7 @@ ARG BUILDPLATFORM
ARG BUILDARCH
# Frontend build stage.
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS frontend-builder
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f AS frontend-builder
# Helpful debug output to see what platforms BuildKit thinks it's using
RUN echo "BUILDPLATFORM=$BUILDPLATFORM BUILDARCH=$BUILDARCH TARGETPLATFORM=$TARGETPLATFORM TARGETARCH=$TARGETARCH"
@@ -25,7 +25,7 @@ COPY src/frontend/ ./
RUN npm run build
# Use python-slim as the base image
FROM python:3.14.6-slim@sha256:b877e50bd90de10af8d82c57a022fc2e0dc731c5320d762a27986facfc3355c1 AS base
FROM python:3.14-slim@sha256:1697e8e8d39bf168e177ac6b5fdab6df86d81cfc24dae17dfb96cfc3ef76b4dd AS base
COPY --from=ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 /uv /uvx /bin/
@@ -72,12 +72,7 @@ RUN apt-get update && \
# --- Tor support (activated via USING_TOR=true) ---
tor \
supervisor \
iptables \
# --- WireGuard support (activated via USING_WIREGUARD=true) ---
wireguard-tools \
iproute2 \
procps \
ca-certificates && \
iptables && \
# Configure iptables alternatives for tor.sh compatibility
update-alternatives --set iptables /usr/sbin/iptables-legacy && \
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
@@ -136,7 +131,7 @@ RUN mkdir -p \
ln -s /tmp/shelfmark/seleniumbase/archived_files /app/archived_files && \
chown -R 1000:1000 /config /books /home/shelfmark /tmp/shelfmark /var/log/shelfmark && \
chmod -R a+rX /app && \
chmod +x /app/entrypoint.sh /app/tor.sh /app/wireguard.sh /app/genDebug.sh
chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
# Expose the application port
EXPOSE ${FLASK_PORT}
@@ -152,39 +147,21 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
FROM base AS shelfmark
# --- Chromium (PINNED to 149.0.7827.196) ---
# Debian's chromium 150.0.7871.46-1~deb13u1 security update (trixie-security,
# 2026-07-05) no longer opens the DevTools remote-debugging TCP port at all
# (no listener, no DevToolsActivePort file, even with a custom --user-data-dir;
# the RemoteDebuggingAllowed policy does not restore it). The SeleniumBase
# Pure-CDP driver connects through that port (/json/version), so with 150 every
# internal bypass dies with "Pure CDP browser startup failed" and all
# CF-gated downloads fail. Install the last working version from
# snapshot.debian.org until the bypasser can talk to Chromium >= 150 (e.g.
# pipe-based DevTools / UC mode) or seleniumbase ships a fix.
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
# This flag is set in internal_bypasser.py _get_browser_args()
ARG CHROMIUM_VERSION=149.0.7827.196-1~deb13u1
ARG CHROMIUM_SNAPSHOT=20260704T000000Z
RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/${CHROMIUM_SNAPSHOT}/ trixie-security main" \
> /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
apt-get update -o Acquire::Retries=5 && \
apt-get install -y --no-install-recommends -o Acquire::Retries=5 \
RUN apt-get update && \
apt-get install -y --no-install-recommends \
# For dumb display
xvfb \
# For screen recording
ffmpeg \
chromium=${CHROMIUM_VERSION} \
chromium-common=${CHROMIUM_VERSION} \
# --- Chromium (unpinned - uses latest from Debian repos) ---
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
# This flag is set in internal_bypasser.py _get_browser_args()
chromium \
chromium-common \
# For tkinter (pyautogui)
python3-tk \
# For RAR extraction
unrar-free && \
# Keep apt from "upgrading" chromium past the pin inside derived images
printf 'Package: chromium chromium-common\nPin: version %s\nPin-Priority: 1001\n' "${CHROMIUM_VERSION}" \
> /etc/apt/preferences.d/chromium-pin && \
rm /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
# Create symlink so rarfile library can find unrar
ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
# Cleanup APT cache
@@ -196,21 +173,6 @@ RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --locked --no-default-groups --extra browser
# Deterministically resolve the Xlib namespace collision.
# pyautogui/mouseinfo pull the stale `python3-xlib` (0.15, 2014), while the
# `--extra browser` set pulls `python-xlib` (0.33). Both packages install into
# the same top-level `Xlib/` namespace, so whichever lands last wins. When the
# 2014 build wins, `Xlib.X` is missing `FamilyServerInterpreted`, which the
# SeleniumBase Pure-CDP driver requires at browser startup -> every bypass fails
# with "module 'Xlib.X' has no attribute 'FamilyServerInterpreted'" and no
# Cloudflare/DDoS-Guard protected download can complete. Drop the stale package
# and force python-xlib 0.33 to own the namespace. pyautogui runs fine against
# 0.33 (superset API).
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip uninstall --python /app/.venv/bin/python python3-xlib && \
uv pip install --python /app/.venv/bin/python --reinstall python-xlib==0.33 && \
/app/.venv/bin/python -c "import Xlib.X; assert hasattr(Xlib.X, 'FamilyServerInterpreted'), 'Xlib.X.FamilyServerInterpreted missing after fix'; print('Xlib namespace OK:', Xlib.__version__)"
# uv is only needed while building the image.
RUN rm -f /usr/bin/uv /usr/bin/uvx
+1 -29
View File
@@ -1,4 +1,4 @@
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov e2e-platform e2e-platform-profile e2e-platform-matrix e2e-platform-full e2e-platform-build checks fix
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov checks fix
# Frontend directory
FRONTEND_DIR := src/frontend
@@ -38,10 +38,6 @@ help:
@echo " python-checks - Run all Python static analysis checks"
@echo " python-test - Run unit tests"
@echo " python-test-cov - Run unit tests with coverage report"
@echo " e2e-platform - Run e2e docker platform (baseline profile)"
@echo " e2e-platform-profile PROFILE=<name> - Run e2e platform for one profile"
@echo " e2e-platform-matrix - Run e2e platform across all config profiles"
@echo " e2e-platform-full - Run heavy 'full' profile (real Chrome bypasser + DoH + real qBittorrent)"
@echo " clean - Remove node_modules and build artifacts"
@echo ""
@echo "Backend (Docker):"
@@ -131,30 +127,6 @@ python-test-cov:
@echo "Running tests with coverage..."
uv run pytest tests/ -x --tb=short -m "not integration and not e2e" --cov --cov-report=term-missing
# E2E docker platform: hermetic stack (mock AA/Cloudflare/bypasser/DNS/proxy/Tor)
# exercised across config profiles. See tests/e2e/platform/README.md.
E2E_PLATFORM_DIR := tests/e2e/platform
e2e-platform:
@echo "Running e2e platform (baseline profile)..."
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/baseline.env
e2e-platform-profile:
@echo "Running e2e platform (profile=$(PROFILE))..."
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/$(PROFILE).env
e2e-platform-matrix:
@echo "Running e2e platform matrix (all profiles)..."
cd $(E2E_PLATFORM_DIR) && ./run-matrix.sh
e2e-platform-build:
@echo "Pre-building e2e platform images once (reused across profiles)..."
cd $(E2E_PLATFORM_DIR) && ./build-images.sh
e2e-platform-full:
@echo "Running e2e platform FULL profile (real Chrome bypasser + DoH + real qBittorrent)..."
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/full.env
# Frontend linting
frontend-lint:
@echo "Running Oxlint..."
-43
View File
@@ -1,43 +0,0 @@
# Routes all traffic through a WireGuard tunnel - requires root startup.
#
# Mount your wg-quick config at /config/wg0.conf (read-only is fine). All
# non-LAN egress is forced through the tunnel by an iptables kill-switch, so if
# the tunnel drops, external traffic fails closed. LAN ranges (WebUI + internal
# download clients like Prowlarr / qBittorrent) stay reachable off-tunnel.
services:
shelfmark-wireguard:
image: ghcr.io/calibrain/shelfmark:latest
environment:
FLASK_PORT: 8084
# Quoted so it is passed as the literal string "true": entrypoint.sh compares
# $USING_WIREGUARD against "true", and some Compose implementations stringify
# a bare YAML boolean as "True", which would silently NOT enable WireGuard.
USING_WIREGUARD: "true"
# Path to the mounted wg-quick config (default shown).
WIREGUARD_CONFIG: /config/wg0.conf
# CIDRs kept OFF the tunnel so the WebUI and internal clients stay reachable.
LAN_NETWORK: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
PUID: 1000
PGID: 1000
cap_add:
- NET_ADMIN
- NET_RAW
# WireGuard needs the module/kernel routing; NET_ADMIN covers wg-quick.
sysctls:
- net.ipv4.conf.all.src_valid_mark=1
# Disable IPv6 in the container so the kill-switch can guarantee no IPv6
# leak path on kernels/containers without a usable ip6tables. wireguard.sh
# fails closed if IPv6 is neither kill-switched nor disabled. If your host
# DOES have a working ip6tables you may omit these (an ip6tables kill-switch
# is installed instead); or set WIREGUARD_ALLOW_IPV6_LEAK=true only if the
# container genuinely has no IPv6 connectivity.
- net.ipv6.conf.all.disable_ipv6=1
- net.ipv6.conf.default.disable_ipv6=1
ports:
- 8084:8084
restart: unless-stopped
volumes:
- /path/to/books:/books # Default destination for book downloads
- /path/to/config:/config # App configuration (put wg0.conf here)
# Required for torrent / usenet - path must match your download client's volume exactly
# - /path/to/downloads:/path/to/downloads
-25
View File
@@ -1,25 +0,0 @@
# Local development - WireGuard variant
services:
shelfmark-wireguard-dev:
extends:
file: ./compose/docker-compose.wireguard.yml
service: shelfmark-wireguard
build:
context: .
dockerfile: Dockerfile
target: shelfmark
environment:
# Quoted so they are passed as the literal string "true" (entrypoint.sh and
# the app compare against "true"); a bare YAML boolean can be stringified as
# "True" by some Compose variants, silently disabling the feature.
DEBUG: "true"
USING_WIREGUARD: "true"
WIREGUARD_CONFIG: /config/wg0.conf
volumes:
- ./.local/config:/config
- ./.local/books:/books
- ./.local/log:/var/log/shelfmark
- ./.local/tmp:/tmp/shelfmark
# Place your wg-quick config at ./.local/config/wg0.conf
# Required for torrent / usenet - path must match your download client's volume exactly
# - /path/to/downloads:/path/to/downloads
-4
View File
@@ -91,10 +91,6 @@ Example:
- Shelfmark can see the same files at `/downloads/books/...`
- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
If the files are copied or synced into Shelfmark on a delay, increase **Completed Path Wait (seconds)**
in Settings -> Advanced. The default is 60 seconds; seedbox or remote-sync setups may need a value
longer than the sync interval.
## File Processing Options
### Transfer Method (Torrent / Usenet Only)
+2 -117
View File
@@ -7,7 +7,6 @@ This document lists all configuration options that can be set via environment va
## Table of Contents
- [Bootstrap Configuration](#bootstrap-configuration)
- [Egress / VPN Routing](#egress--vpn-routing)
- [General](#general)
- [Search Mode](#search-mode)
- [Downloads](#downloads)
@@ -146,98 +145,6 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme
</details>
## Egress / VPN Routing
These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `USING_WIREGUARD` | Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR. | boolean | `false` |
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick configuration file. | string (path) | `/config/wg0.conf` |
| `WIREGUARD_INTERFACE` | WireGuard interface name brought up by wg-quick. | string | `wg0` |
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable. | string (comma-separated) | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
| `WIREGUARD_ENFORCE_DNS` | Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable. | boolean | `true` |
| `WIREGUARD_DNS` | Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead. | string (comma-separated) | `unset (uses config DNS = line)` |
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface. | boolean | `true` |
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel. | boolean | `false` |
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK). | boolean | `false` |
| `WIREGUARD_STALE_AFTER` | Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel. | number | `180` |
<details>
<summary>Detailed descriptions</summary>
#### `USING_WIREGUARD`
Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.
- **Type:** boolean
- **Default:** `false`
#### `WIREGUARD_CONFIG`
Path to the mounted wg-quick configuration file.
- **Type:** string (path)
- **Default:** `/config/wg0.conf`
#### `WIREGUARD_INTERFACE`
WireGuard interface name brought up by wg-quick.
- **Type:** string
- **Default:** `wg0`
#### `LAN_NETWORK`
Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.
- **Type:** string (comma-separated)
- **Default:** `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`
#### `WIREGUARD_ENFORCE_DNS`
Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.
- **Type:** boolean
- **Default:** `true`
#### `WIREGUARD_DNS`
Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.
- **Type:** string (comma-separated)
- **Default:** `unset (uses config DNS = line)`
#### `WIREGUARD_DISABLE_IPV6`
Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.
- **Type:** boolean
- **Default:** `true`
#### `WIREGUARD_ALLOW_IPV6_LEAK`
Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.
- **Type:** boolean
- **Default:** `false`
#### `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL`
When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).
- **Type:** boolean
- **Default:** `false`
#### `WIREGUARD_STALE_AFTER`
Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.
- **Type:** number
- **Default:** `180`
</details>
## General
| Variable | Description | Type | Default |
@@ -316,7 +223,6 @@ Default language filter for searches.
| `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` |
| `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` |
| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` |
| `FORCE_COMBINED_SEARCH` | Force combined search whenever it's available. Locks the combined toggle on. | boolean | `false` |
| `METADATA_PROVIDER` | Choose which metadata provider to use for book searches. | string (choice) | `openlibrary` |
| `METADATA_PROVIDER_AUDIOBOOK` | Metadata provider for audiobook searches. Uses the book provider if not set. | string (choice) | _empty string_ |
| `METADATA_PROVIDER_COMBINED` | Metadata provider for combined mode searches. Uses the book provider if not set. | string (choice) | _empty string_ |
@@ -364,15 +270,6 @@ Show the option to search for and download both a book and audiobook together.
- **Type:** boolean
- **Default:** `true`
#### `FORCE_COMBINED_SEARCH`
**Always Use Combined Search**
Force combined search whenever it's available. Locks the combined toggle on.
- **Type:** boolean
- **Default:** `false`
#### `METADATA_PROVIDER`
**Book Metadata Provider**
@@ -1051,7 +948,6 @@ Comma-separated hosts to bypass proxy (e.g., localhost,127.0.0.1,10.*,*.local)
| `CUSTOM_SCRIPT` | Path to a script to run after each successful download. Must be executable. | string | _none_ |
| `CUSTOM_SCRIPT_PATH_MODE` | Pass the path to the custom script as an absolute path or relative to the destination folder. | string (choice) | `absolute` |
| `CUSTOM_SCRIPT_JSON_PAYLOAD` | Send a JSON payload to the script via stdin. Useful for multi-file imports (audiobooks) or richer metadata without relying on path parsing. | boolean | `false` |
| `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT` | How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows. | number | `60` |
| `COVERS_CACHE_ENABLED` | Cache book covers on the server for faster loading. | boolean | `true` |
| `COVERS_CACHE_TTL` | How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork). | number | `0` |
| `COVERS_CACHE_MAX_SIZE_MB` | Maximum disk space for cached covers. Oldest images are removed when limit is reached. | number | `500` |
@@ -1132,16 +1028,6 @@ Send a JSON payload to the script via stdin. Useful for multi-file imports (audi
- **Type:** boolean
- **Default:** `false`
#### `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT`
**Completed Path Wait (seconds)**
How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows.
- **Type:** number
- **Default:** `60`
- **Constraints:** min: 0, max: 3600
#### `COVERS_CACHE_ENABLED`
**Enable Cover Cache**
@@ -1397,7 +1283,7 @@ Delay between requests in seconds to avoid rate limiting (0-10).
| `IRC_USE_TLS` | Enable TLS/SSL encryption for the IRC connection. Disable for servers that don't support TLS. | boolean | `true` |
| `IRC_CHANNEL` | Channel name without the # prefix | string | _none_ |
| `IRC_NICK` | Your IRC nickname (required). Must be unique on the IRC network. | string | _none_ |
| `IRC_SEARCH_BOT` | The search bot to address queries to (required). | string | _none_ |
| `IRC_SEARCH_BOT` | The search bot to query for results | string | _none_ |
| `IRC_CACHE_TTL` | How long to keep cached search results before they expire. | string (choice) | `2592000` |
<details>
@@ -1455,11 +1341,10 @@ Your IRC nickname (required). Must be unique on the IRC network.
**Search bot**
The search bot to address queries to (required). Searches are sent as "@<bot> <query>". Without it, queries would be posted unaddressed to the channel.
The search bot to query for results
- **Type:** string
- **Default:** _none_
- **Required:** Yes
#### `IRC_CACHE_TTL`
+1 -8
View File
@@ -19,7 +19,7 @@ http://your-server:8084/?q=harry+potter
| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
| `format` | Filter by file format | `/?format=epub` |
| `content` | Filter by content type | `/?content=fiction` |
| `content_type` | Select media type (`ebook`, `audiobook`, or `combined`) in Universal mode only | `/?q=dune&content_type=audiobook` |
| `content_type` | Select media type (`ebook` or `audiobook`) in Universal mode only | `/?q=dune&content_type=audiobook` |
| `sort` | Sort order for results | `/?sort=newest` |
## Multiple Values
@@ -63,11 +63,6 @@ Some parameters support multiple values by repeating the parameter:
/?q=dune&content_type=audiobook
```
**Universal search forcing combined (ebook + audiobook):**
```
/?q=dune&content_type=combined
```
## Search Mode Behavior
### Direct Mode
@@ -79,8 +74,6 @@ When Search Mode is set to Direct, all parameters are used to filter results fro
`q`, `sort`, and `content_type` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
`content_type=combined` forces combined mode (search ebook and audiobook providers together), overriding the last-used preference. It is silently ignored if combined mode is unavailable (e.g. the combined selector is disabled in settings, or either content type is blocked by request policy).
## Notes
- URL parameters are read once on page load
+1 -27
View File
@@ -81,13 +81,6 @@ if is_truthy "$ENABLE_LOGGING_VALUE"; then
fi
fi
# Egress modes are mutually exclusive. Check this BEFORE starting either one so
# we never run tor.sh and then abort, leaving a half-configured network stack.
if [ "$USING_TOR" = "true" ] && [ "$USING_WIREGUARD" = "true" ]; then
echo "USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode." >&2
exit 1
fi
if [ "$USING_TOR" = "true" ]; then
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
echo "USING_TOR=true requires the container to start as root." >&2
@@ -97,15 +90,6 @@ if [ "$USING_TOR" = "true" ]; then
./tor.sh
fi
if [ "$USING_WIREGUARD" = "true" ]; then
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
echo "USING_WIREGUARD=true requires the container to start as root." >&2
echo "Non-root mode skips the privileged network setup WireGuard depends on." >&2
exit 1
fi
./wireguard.sh
fi
if [ "$FILE_LOGGING_ENABLED" = "true" ]; then
start_file_logging "$LOG_FILE"
fi
@@ -251,11 +235,7 @@ test_write() {
return 1
fi
# This is a probe: a failure here is expected (e.g. a fresh root-owned bind
# mount) and is recovered by the caller via change_ownership + re-probe. Hide
# the shell's "Permission denied"/"Read-only file system" stderr so a handled
# probe miss doesn't masquerade as a real boot failure in the logs.
if ! run_as_target_user sh -c 'echo 0123456789_TEST 2>/dev/null > "$1"' _ "$test_file"; then
if ! run_as_target_user sh -c 'echo 0123456789_TEST > "$1"' _ "$test_file"; then
echo "Failed to write test file in $folder as $USERNAME"
return 1
fi
@@ -468,12 +448,6 @@ else
if [ $config_ok -ne 0 ]; then
fail_unwritable_config_dir "$CONFIG_PATH"
fi
# The ingest/destination library (default /books) is user data and may be a
# bind mount owned by another uid; downloads fail with "Destination not
# writable" if the runtime user can't write there. Fix the top-level dir only
# (root mode) so we don't recursively chown a potentially huge library.
make_writable "${INGEST_DIR:-/books}" root
fi
# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
+4 -4
View File
@@ -22,25 +22,25 @@ dependencies = [
"qbittorrent-api",
"transmission-rpc",
"authlib>=1.7.2,<1.8",
"apprise>=1.12.0",
"apprise>=1.10.0",
]
[project.optional-dependencies]
browser = [
"pyvirtualdisplay",
"pyautogui",
"seleniumbase==4.50.5",
"seleniumbase==4.48.4",
"python-xlib",
]
[dependency-groups]
dev = [
"basedpyright>=1.39.9",
"basedpyright>=1.39.3",
"prek",
"pytest",
"pytest-cov",
"pytest-xdist>=3.8.0",
"ruff==0.15.20",
"ruff==0.15.12",
"vulture>=2.14",
]
-31
View File
@@ -2,9 +2,6 @@
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
> [!NOTE]
> This project is in a stable state as of May 2026 but is not under active maintenance.
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
Works great alongside the following library tools, with support for automatic imports:
@@ -106,16 +103,6 @@ Environment variables work for initial setup and Docker deployments. They serve
| `PUID` / `PGID` | Runtime user/group for the default root-startup flow (also supports legacy `UID`/`GID`) | `1000` / `1000` |
| `SEARCH_MODE` | `direct` or `universal` | `universal` |
| `USING_TOR` | Enable Tor routing (requires root startup) | `false` |
| `USING_WIREGUARD` | Enable WireGuard VPN egress with kill-switch (requires root startup) | `false` |
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick config | `/config/wg0.conf` |
| `WIREGUARD_INTERFACE` | WireGuard interface name | `wg0` |
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI / internal clients stay reachable | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
| `WIREGUARD_ENFORCE_DNS` | Pin the resolver (via `WIREGUARD_DNS`, else the config's `DNS =`) so DNS can't silently fall back to an off-tunnel path. Designed for a trusted LAN resolver kept reachable via `LAN_NETWORK` (query leaves over the LAN; download still egresses via the tunnel) — it does **not** force queries through the tunnel. Docker's embedded resolver (`127.0.0.11`) is preserved when present so container-name resolution keeps working; pin its upstream via the container's `dns:` list. Fails closed if no resolver is available or `/etc/resolv.conf` is not writable. | `true` |
| `WIREGUARD_DNS` | Explicit resolver(s) to pin (comma/space separated). Use when the VPN's pushed DNS filters domains you need; point at a resolver reachable via the tunnel or an allowed LAN resolver. | _(unset; uses config `DNS =`)_ |
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 from the tunnel config (many container kernels lack the ip6tables `raw` table wg-quick needs) and remove IPv6 as a leak surface. | `true` |
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue even when an IPv6 kill-switch can't be installed AND IPv6 can't be disabled. Only set if the container has no IPv6 connectivity. | `false` |
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | Opt-in off-tunnel WebUI reachability. Default (`false`) keeps the kill-switch strictly fail-closed: the only off-tunnel egress is loopback, the tunnel device and the LAN allowlist. Set `true` only if a **non-LAN** client (e.g. a public reverse proxy on another segment) must reach the WebUI; it permits app-server **replies** (`--sport FLASK_PORT`, conntrack REPLY) off-tunnel — server replies only, never client-initiated egress. LAN clients never need it (covered by `LAN_NETWORK`). | `false` |
| `WIREGUARD_STALE_AFTER` | Seconds since the last handshake before the healthcheck bounces the tunnel. | `180` |
See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
@@ -151,24 +138,6 @@ docker compose -f docker-compose.tor.yml up -d
- Timezone is auto-detected from Tor exit node
- Custom DNS/proxy settings are ignored when Tor is active
#### WireGuard VPN Routing
Optional WireGuard support to route all external egress through a VPN tunnel with a fail-closed kill-switch:
```bash
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.wireguard.yml
# place your wg-quick config where the compose mounts /config, as wg0.conf
docker compose -f docker-compose.wireguard.yml up -d
```
**Notes:**
- Requires root startup
- Requires `NET_ADMIN` and `NET_RAW` capabilities
- Mount a standard wg-quick config at `WIREGUARD_CONFIG` (default `/config/wg0.conf`)
- All non-LAN egress is forced through the tunnel; if the tunnel drops, external traffic **fails closed** while LAN ranges (WebUI, Prowlarr, qBittorrent) stay reachable
- IPv4 and IPv6 both fail closed. On kernels without a usable `ip6tables`, disable IPv6 for the container (`sysctls: net.ipv6.conf.all.disable_ipv6=1`, as in the compose example) or the container refuses to start rather than risk an IPv6 leak
- A supervised healthcheck bounces the tunnel if the handshake goes stale, and refreshes the endpoint allow rules so a roaming/rotated peer endpoint can reconnect
- Mutually exclusive with `USING_TOR`
- **DNS trust:** `WIREGUARD_DNS` must be a resolver you trust on a trusted network segment. When it is a LAN resolver (kept reachable off-tunnel by `LAN_NETWORK`), the query to that resolver leaves as plaintext UDP/53 on the LAN — the resolver is responsible for encrypting upstream. Two resolver paths exist: (1) when Docker's embedded resolver (`127.0.0.11`) is present it is **preserved** so container names (Prowlarr, qBittorrent) resolve — you MUST pin its upstream to a trusted resolver via the container's compose `dns:` list, since `WIREGUARD_DNS` cannot repoint the embedded resolver from inside the container; (2) otherwise `WIREGUARD_DNS`/the config `DNS =` line is written to `/etc/resolv.conf`. Setting `WIREGUARD_ENFORCE_DNS=false` is a **foot-gun**: with no embedded resolver present the container then uses its inherited resolver, which forwards to the Docker daemon's upstream **off-tunnel**, leaking your DNS. Leave enforcement on unless you have pinned the resolver another way.
### Lite
A lighter image without the built-in browser automation. Ideal for:
-111
View File
@@ -238,113 +238,6 @@ def _generate_bootstrap_env_docs() -> list[str]:
return lines
def _generate_egress_env_docs() -> list[str]:
"""Generate documentation for VPN/Tor egress environment variables.
These are startup-only variables consumed by entrypoint.sh / wireguard.sh
(before and outside the settings registry) to select and configure the
transparent-egress kill-switch. `USING_TOR` has a registry-backed entry
under Network and is cross-referenced rather than repeated here so the two
mutually exclusive egress modes are discoverable side by side without
emitting a duplicate `#### USING_TOR` anchor.
"""
egress_vars = [
{
"name": "USING_WIREGUARD",
"description": "Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.",
"type": "boolean",
"default": "false",
},
{
"name": "WIREGUARD_CONFIG",
"description": "Path to the mounted wg-quick configuration file.",
"type": "string (path)",
"default": "/config/wg0.conf",
},
{
"name": "WIREGUARD_INTERFACE",
"description": "WireGuard interface name brought up by wg-quick.",
"type": "string",
"default": "wg0",
},
{
"name": "LAN_NETWORK",
"description": "Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.",
"type": "string (comma-separated)",
"default": "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
},
{
"name": "WIREGUARD_ENFORCE_DNS",
"description": "Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.",
"type": "boolean",
"default": "true",
},
{
"name": "WIREGUARD_DNS",
"description": "Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.",
"type": "string (comma-separated)",
"default": "unset (uses config DNS = line)",
},
{
"name": "WIREGUARD_DISABLE_IPV6",
"description": "Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.",
"type": "boolean",
"default": "true",
},
{
"name": "WIREGUARD_ALLOW_IPV6_LEAK",
"description": "Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.",
"type": "boolean",
"default": "false",
},
{
"name": "WIREGUARD_ALLOW_WEBUI_OFFTUNNEL",
"description": "When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).",
"type": "boolean",
"default": "false",
},
{
"name": "WIREGUARD_STALE_AFTER",
"description": "Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.",
"type": "number",
"default": "180",
},
]
lines = [
"## Egress / VPN Routing",
"",
"These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.",
"",
"| Variable | Description | Type | Default |",
"|----------|-------------|------|---------|",
]
lines.extend(
f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |"
for var in egress_vars
)
lines.append("")
lines.append("<details>")
lines.append("<summary>Detailed descriptions</summary>")
lines.append("")
for var in egress_vars:
lines.append(f"#### `{var['name']}`")
lines.append("")
lines.append(var["description"])
lines.append("")
lines.append(f"- **Type:** {var['type']}")
lines.append(f"- **Default:** `{var['default']}`")
lines.append("")
lines.append("</details>")
lines.append("")
return lines
def generate_env_docs() -> str:
"""Generate markdown documentation for all environment variables."""
# Import settings modules to ensure all settings are registered
@@ -389,7 +282,6 @@ def generate_env_docs() -> str:
# Generate TOC
toc_entries = [
"- [Bootstrap Configuration](#bootstrap-configuration)",
"- [Egress / VPN Routing](#egress--vpn-routing)",
]
# Ungrouped tabs first
@@ -415,9 +307,6 @@ def generate_env_docs() -> str:
# Add bootstrap environment variables documentation
lines.extend(_generate_bootstrap_env_docs())
# Add egress / VPN routing (startup-only, shell-driven) documentation
lines.extend(_generate_egress_env_docs())
# Generate documentation for ungrouped tabs
for tab in grouped_tabs.get(None, []):
lines.extend(_generate_tab_docs(tab))
+1 -39
View File
@@ -939,17 +939,7 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
result_path = (
Path(tempfile.gettempdir()) / f"shelfmark-bypass-{os.getpid()}-{time.time_ns()}.json"
)
# DNS provider state lives only in the parent's memory (no disk persistence), so the
# freshly spawned helper would otherwise pre-resolve AA hostnames against the system
# resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
# active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
# resolver rules.
payload = {
"url": url,
"retry": retry,
"result_path": str(result_path),
"dns_config": network.get_dns_config(),
}
payload = {"url": url, "retry": retry, "result_path": str(result_path)}
env_vars = os.environ.copy()
env_vars[_BYPASS_CHILD_ENV] = "1"
env_vars = _prepare_child_browser_env(env_vars)
@@ -1288,30 +1278,6 @@ def get_bypassed_page(
return response_html
def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
"""Mirror the parent process's active DNS provider in this helper subprocess.
DNS state is in-memory only, so a fresh helper defaults to system DNS and would
pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
the same DoH/custom resolver the parent already validated.
"""
provider = str(dns_config.get("provider") or "").strip().lower()
# "auto" means the parent has not rotated off system DNS yet, so the helper's own
# default initialization already matches it - nothing to override.
if not provider or provider == "auto":
return
manual_servers = dns_config.get("servers") if provider == "manual" else None
try:
network.set_dns_provider(
provider,
manual_servers,
use_doh=bool(dns_config.get("doh_enabled")),
)
except (OSError, RuntimeError, ValueError) as exc:
logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
def _run_child_process() -> int:
"""CLI entrypoint used by the Docker helper subprocess."""
request = json.loads(sys.stdin.read() or "{}")
@@ -1321,10 +1287,6 @@ def _run_child_process() -> int:
request.get("retry"), _coerce_positive_int(app_config.MAX_RETRY, 10)
)
dns_config = request.get("dns_config")
if isinstance(dns_config, dict):
_apply_parent_dns_config(dns_config)
try:
html = get(url, retry=retry)
payload = {
+1 -2
View File
@@ -2,7 +2,6 @@
from __future__ import annotations
import hashlib
from typing import Any
from shelfmark.core.config import config
@@ -24,7 +23,7 @@ _BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
return f"{base_url}|{username}|{hashlib.sha256(password.encode()).hexdigest()}"
return f"{base_url}|{username}|{hash(password)}"
def _get_booklore_select_options(
-68
View File
@@ -36,9 +36,6 @@ from shelfmark.core.settings_registry import (
register_settings,
)
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
"""Validate advanced settings before persisting."""
@@ -46,40 +43,6 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
logger = setup_logger(__name__)
timeout_key = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
if timeout_key in values:
raw_timeout = values.get(timeout_key)
if isinstance(raw_timeout, bool):
return {
"error": True,
"message": "Completed Path Wait must be a number of seconds",
"values": values,
}
if raw_timeout is None:
return {
"error": True,
"message": "Completed Path Wait must be a number of seconds",
"values": values,
}
try:
timeout_seconds = int(raw_timeout)
except TypeError, ValueError:
return {
"error": True,
"message": "Completed Path Wait must be a number of seconds",
"values": values,
}
if timeout_seconds < 0 or timeout_seconds > _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX:
return {
"error": True,
"message": (
"Completed Path Wait must be between 0 and "
f"{_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX} seconds"
),
"values": values,
}
values[timeout_key] = timeout_seconds
mappings = values.get("PROWLARR_REMOTE_PATH_MAPPINGS")
if mappings is None:
return {"error": False, "values": values}
@@ -490,14 +453,6 @@ def search_mode_settings() -> list[SettingsField]:
show_when={"field": "SEARCH_MODE", "value": "universal"},
user_overridable=True,
),
CheckboxField(
key="FORCE_COMBINED_SEARCH",
label="Always Use Combined Search",
description="Force combined search whenever it's available. Locks the combined toggle on.",
default=False,
show_when={"field": "SEARCH_MODE", "value": "universal"},
user_overridable=True,
),
HeadingField(
key="universal_mode_heading",
title="Universal Mode Settings",
@@ -1485,17 +1440,6 @@ def download_source_settings() -> list[SettingsField]:
),
default=False,
),
CheckboxField(
key="DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH",
label="Detect Language From Distant Path",
description=(
"When language metadata is missing or unknown, parse the distant path "
"(file path shown in search results) for language tags like [BD FR] or [En]. "
"Also enables local language filtering so lgli files without AA language "
"metadata are not excluded before the distant path can be checked."
),
default=False,
),
PasswordField(
key="AA_DONATOR_KEY",
label="Account Donator Key",
@@ -1817,18 +1761,6 @@ def advanced_settings() -> list[SettingsField]:
title="Remote Path Mappings",
description="Map download client paths to paths inside Shelfmark. Needed when volume mounts differ between containers.",
),
NumberField(
key="DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT",
label="Completed Path Wait (seconds)",
description=(
"How long to wait after a torrent or usenet client reports completion "
"for the completed file path to become visible to Shelfmark. Increase "
"this for seedbox or remote-sync workflows."
),
default=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT,
min_value=0,
max_value=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX,
),
TableField(
key="PROWLARR_REMOTE_PATH_MAPPINGS",
label="Path Mappings",
-6
View File
@@ -82,7 +82,6 @@ _SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
"DEFAULT_RELEASE_SOURCE",
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
"SHOW_COMBINED_SELECTOR",
"FORCE_COMBINED_SEARCH",
*_SEARCH_PREFERENCE_PROVIDER_KEYS,
}
@@ -224,11 +223,6 @@ def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | N
return value, None
return bool(value), None
if key == "FORCE_COMBINED_SEARCH":
if isinstance(value, bool):
return value, None
return bool(value), None
return value, None
-42
View File
@@ -393,41 +393,6 @@ def _plugin_label(plugin: object, fallback_scheme: str) -> str:
return " ".join(parts)
def _apprise_proxy_env() -> dict[str, str]:
"""Build proxy env vars from app config so Apprise respects the proxy setting."""
import os
from shelfmark.core.config import config as _cfg
mode = str(_cfg.get("PROXY_MODE", "") or "").lower()
env: dict[str, str] = {}
if mode == "http":
http = str(_cfg.get("HTTP_PROXY", "") or "").strip()
https = str(_cfg.get("HTTPS_PROXY", "") or "").strip() or http
if http:
env["HTTP_PROXY"] = http
env["http_proxy"] = http
if https:
env["HTTPS_PROXY"] = https
env["https_proxy"] = https
elif mode == "socks5":
socks = str(_cfg.get("SOCKS5_PROXY", "") or "").strip()
if socks:
env["HTTP_PROXY"] = socks
env["http_proxy"] = socks
env["HTTPS_PROXY"] = socks
env["https_proxy"] = socks
no_proxy = str(_cfg.get("NO_PROXY", "") or "").strip()
if no_proxy and env:
env["NO_PROXY"] = no_proxy
env["no_proxy"] = no_proxy
# Don't override if the user already set these in the environment directly
return {k: v for k, v in env.items() if not os.environ.get(k)}
def _dispatch_to_apprise(
urls: Iterable[str],
*,
@@ -435,8 +400,6 @@ def _dispatch_to_apprise(
body: str,
notify_type: object,
) -> dict[str, Any]:
import os
normalized_urls = _normalize_urls(list(urls))
url_schemes = _extract_url_schemes(normalized_urls)
if not normalized_urls:
@@ -445,11 +408,6 @@ def _dispatch_to_apprise(
if apprise is None:
return {"success": False, "message": "Apprise is not installed"}
proxy_env = _apprise_proxy_env()
if proxy_env:
logger.debug("Applying proxy env for Apprise dispatch: %s", list(proxy_env.keys()))
os.environ.update(proxy_env)
valid_urls = 0
invalid_urls = 0
delivered_urls = 0
-7
View File
@@ -52,13 +52,6 @@ def normalize_http_url(
if scheme:
normalized = f"{scheme}://{normalized}"
# Strip query string and fragment — mirrors are used as base URLs for
# constructing search requests; params/fragments on the configured URL
# produce malformed URLs when paths are appended (issue #999).
parsed = urlparse(normalized)
if parsed.query or parsed.fragment:
normalized = parsed._replace(query="", fragment="").geturl()
if strip_trailing_slash:
normalized = normalized.rstrip("/")
+12 -103
View File
@@ -3,7 +3,6 @@
from __future__ import annotations
import errno
import math
import shutil
import time
from abc import ABC, abstractmethod
@@ -56,19 +55,6 @@ SECONDS_PER_HOUR = 3600
# How long to wait for completed files to appear (seconds)
COMPLETED_PATH_RETRY_INTERVAL = 5
COMPLETED_PATH_MAX_ATTEMPTS = 12 # 12 attempts * 5s = 60s grace period
COMPLETED_PATH_TIMEOUT_SETTING = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
COMPLETED_PATH_TIMEOUT_MAX_SECONDS = 3600
_RETRYABLE_COMPLETED_PATH_ERRNOS = frozenset(
code
for code in (
errno.ENOENT,
getattr(errno, "ESTALE", None),
getattr(errno, "EAGAIN", None),
getattr(errno, "EBUSY", None),
getattr(errno, "ETIMEDOUT", None),
)
if code is not None
)
@dataclass(frozen=True)
@@ -83,39 +69,6 @@ class DownloadRequest:
ratio_limit: float | None = None
@dataclass(frozen=True)
class _CompletedPathResolution:
path: Path | None
error: str | None
retryable: bool
def _coerce_completed_path_timeout_seconds(value: object, default: float) -> float:
if isinstance(value, bool) or value is None:
return default
if isinstance(value, (int, float)):
parsed = float(value)
elif isinstance(value, str):
try:
parsed = float(value.strip())
except ValueError:
return default
else:
return default
if not math.isfinite(parsed) or parsed < 0:
return default
return min(parsed, float(COMPLETED_PATH_TIMEOUT_MAX_SECONDS))
def _is_retryable_completed_path_probe(error: OSError | None) -> bool:
return error is not None and error.errno in _RETRYABLE_COMPLETED_PATH_ERRNOS
def _path_needs_mapping(path: str) -> bool:
return (len(path) >= WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":") or "\\" in path
def _diagnose_path_issue(path: str) -> str:
"""Analyze a path and return diagnostic hints for common issues.
@@ -214,12 +167,6 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Maximum attempts when waiting for completed files."""
return COMPLETED_PATH_MAX_ATTEMPTS
def _completed_path_timeout_seconds(self) -> float:
"""Total time to wait for completed files to appear on disk."""
fallback = self._completed_path_retry_interval() * self._completed_path_max_attempts()
configured = config.get(COMPLETED_PATH_TIMEOUT_SETTING, fallback)
return _coerce_completed_path_timeout_seconds(configured, fallback)
def _get_category_for_task(self, client: DownloadClient, task: DownloadTask) -> str | None:
"""Get audiobook category if configured and applicable, else None for default."""
if not is_audiobook(task.content_type):
@@ -440,21 +387,6 @@ class ExternalClientHandler(DownloadHandler, ABC):
log_details: bool,
) -> tuple[Path | None, str | None]:
"""Resolve and validate the completed download path once."""
result = self._resolve_download_path_once_detailed(
client,
download_id,
log_details=log_details,
)
return result.path, result.error
def _resolve_download_path_once_detailed(
self,
client: DownloadClient,
download_id: str,
*,
log_details: bool,
) -> _CompletedPathResolution:
"""Resolve and validate a completed path, including retryability."""
try:
raw_path = client.get_download_path(download_id)
except Exception as e:
@@ -470,7 +402,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.debug(
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
)
return _CompletedPathResolution(None, message, retryable=False)
return None, message
if not raw_path:
message = (
@@ -485,7 +417,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.debug(
"Download client returned empty path for %s %s", client.name, download_id
)
return _CompletedPathResolution(None, message, retryable=False)
return None, message
from shelfmark.core.path_mappings import (
get_client_host_identifier,
@@ -525,7 +457,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.error(failure_log, *failure_args)
else:
logger.debug(failure_log, *failure_args)
return _CompletedPathResolution(None, message, retryable=False)
return None, message
remapped_exists, remapped_error = _probe_completed_path(remapped)
@@ -548,7 +480,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
source_path_obj,
remapped,
)
return _CompletedPathResolution(remapped, None, retryable=False)
return remapped, None
message = (
f"Remapped path '{remapped}' does not exist. "
@@ -567,11 +499,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.error(failure_log, *failure_args)
else:
logger.debug(failure_log, *failure_args)
return _CompletedPathResolution(
None,
message,
retryable=_is_retryable_completed_path_probe(remapped_error),
)
return None, message
source_exists, source_error = _probe_completed_path(source_path_obj)
@@ -594,7 +522,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
download_id,
source_path_obj,
)
return _CompletedPathResolution(source_path_obj, None, retryable=False)
return source_path_obj, None
hint = _diagnose_path_issue(raw_path)
if mappings:
@@ -627,12 +555,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.error(failure_log, *failure_args)
else:
logger.debug(failure_log, *failure_args)
return _CompletedPathResolution(
None,
message,
retryable=not _path_needs_mapping(raw_path)
and _is_retryable_completed_path_probe(source_error),
)
return None, message
def _wait_for_completed_path(
self,
@@ -644,37 +567,23 @@ class ExternalClientHandler(DownloadHandler, ABC):
) -> tuple[Path | None, str | None]:
"""Wait briefly for completed files to appear on disk."""
last_error: str | None = None
max_attempts = self._completed_path_max_attempts()
retry_interval = self._completed_path_retry_interval()
timeout_seconds = self._completed_path_timeout_seconds()
if retry_interval <= 0 or timeout_seconds <= 0:
max_attempts = 1
else:
max_attempts = int(math.ceil(timeout_seconds / retry_interval)) + 1
for attempt in range(1, max_attempts + 1):
if cancel_flag and cancel_flag.is_set():
return None, last_error
log_details = attempt == max_attempts
result = self._resolve_download_path_once_detailed(
resolved_path, error = self._resolve_download_path_once(
client,
download_id,
log_details=log_details,
)
if result.path:
return result.path, None
if resolved_path:
return resolved_path, None
last_error = result.error
if not result.retryable:
if not log_details:
logger.error(
"Completed path resolution is not retryable for %s (%s): %s",
client.name,
download_id,
last_error,
)
return None, last_error
last_error = error
if attempt < max_attempts:
status_callback("locating", "Waiting for completed files...")
+4 -54
View File
@@ -258,7 +258,7 @@ class QBittorrentClient(DownloadClient):
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
def _get_torrents_info(
self, torrent_hash: str | None = None, category: str | None = None
self, torrent_hash: str | None = None
) -> tuple[list[SimpleNamespace], str | None]:
"""Get torrent info using GET.
@@ -267,7 +267,6 @@ class QBittorrentClient(DownloadClient):
- Keep "API/auth/connect" errors distinct from "torrent missing".
- If a hash-specific query returns empty, fall back to listing by category
and matching locally.
- Without a hash, `category` narrows the listing to that category.
Returns:
(torrents, error_message)
@@ -302,8 +301,6 @@ class QBittorrentClient(DownloadClient):
primary_params: dict[str, str] = {}
if torrent_hash:
primary_params["hashes"] = torrent_hash
elif category:
primary_params["category"] = category
response = do_request(primary_params)
torrents, error = parse_response(response, request_params=primary_params)
@@ -361,44 +358,6 @@ class QBittorrentClient(DownloadClient):
else:
return torrents, None
def _list_category_hashes(self, category: str | None) -> set[str] | None:
"""Snapshot the hashes qBittorrent currently reports for a category."""
torrents, error = self._get_torrents_info(category=category)
if error:
logger.debug("Could not snapshot qBittorrent torrents: %s", error)
return None
return {str(torrent.hash).lower() for torrent in torrents if getattr(torrent, "hash", None)}
def _discover_added_torrent_hash(
self,
name: str,
category: str | None,
known_hashes: set[str] | None,
) -> str | None:
"""Recover the hash of a torrent that was added without a known info_hash.
A `known_hashes` of None means the pre-add snapshot failed, so only a
torrent matching the requested rename can identify the new arrival.
"""
for _ in range(20):
torrents, error = self._get_torrents_info(category=category)
if error:
logger.debug("qBittorrent hash discovery: %s", error)
else:
new_torrents = [
torrent
for torrent in torrents
if getattr(torrent, "hash", None)
and (known_hashes is None or str(torrent.hash).lower() not in known_hashes)
]
for torrent in new_torrents:
if getattr(torrent, "name", None) == name:
return str(torrent.hash).lower()
if known_hashes is not None and len(new_torrents) == 1:
return str(new_torrents[0].hash).lower()
time.sleep(0.5)
return None
@staticmethod
def is_configured() -> bool:
"""Check if qBittorrent is configured and selected as the torrent client."""
@@ -466,10 +425,6 @@ class QBittorrentClient(DownloadClient):
expected_hash = torrent_info.info_hash
torrent_data = torrent_info.torrent_data
known_hashes: set[str] | None = None
if not expected_hash:
known_hashes = self._list_category_hashes(category)
# Per-torrent seeding limits from indexer
seeding_time_limit_value = kwargs.get("seeding_time_limit")
seeding_time_limit = coerce_optional_int(seeding_time_limit_value)
@@ -504,17 +459,12 @@ class QBittorrentClient(DownloadClient):
result_text = _normalize_add_result(result)
logger.debug("qBittorrent add result: %s", result_text)
if _is_explicit_add_failure(result):
_raise_runtime_error(f"Failed to add torrent: {result_text}")
if not expected_hash:
# qBittorrent fetches .torrent URLs itself, so the add can succeed
# even when no hash could be extracted up front. Recover it by
# watching for the new torrent to appear.
expected_hash = self._discover_added_torrent_hash(name, category, known_hashes)
if not expected_hash:
_raise_runtime_error("Could not determine torrent hash from URL")
if _is_explicit_add_failure(result):
_raise_runtime_error(f"Failed to add torrent: {result_text}")
# Some qBittorrent-compatible clients return HTTP 200 with an empty body
# instead of qBittorrent's literal "Ok." response. Prefer verifying that
# the torrent becomes visible over trusting the response body alone.
+1 -66
View File
@@ -4,7 +4,6 @@ Uses xmlrpc to communicate with rTorrent's RPC interface.
"""
import ssl
import time
import xmlrpc.client as stdlib_xmlrpc_client
from typing import Any, NoReturn, Protocol, cast
from urllib.parse import urlparse
@@ -116,7 +115,6 @@ class RTorrentClient(DownloadClient):
self._rpc = _create_rtorrent_server_proxy(self._base_url)
self._download_dir = config_text(config.get("RTORRENT_DOWNLOAD_DIR", ""))
self._label = config_text(config.get("RTORRENT_LABEL", ""))
self._audiobook_label = config_text(config.get("RTORRENT_AUDIOBOOK_LABEL", ""))
@staticmethod
def is_configured() -> bool:
@@ -161,17 +159,9 @@ class RTorrentClient(DownloadClient):
try:
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
known_hashes: set[str] | None = None
if not (torrent_info.info_hash or expected_hash):
known_hashes = self._list_torrent_hashes()
commands = []
is_audiobook = kwargs.get("content_type") == "audiobook"
default_label = (
self._audiobook_label if is_audiobook and self._audiobook_label else self._label
)
label = category or default_label
label = category or self._label
if label:
logger.debug("Setting rTorrent label: %s", label)
commands.append(f"d.custom1.set={label}")
@@ -200,11 +190,6 @@ class RTorrentClient(DownloadClient):
self._rpc.load.start("", add_url, ";".join(commands))
torrent_hash = torrent_info.info_hash or expected_hash
if not torrent_hash:
# rTorrent fetches .torrent URLs itself, so the add can succeed
# even when no hash could be extracted up front. Recover it by
# watching for the new download to appear.
torrent_hash = self._discover_added_torrent_hash(name, label, known_hashes)
if not torrent_hash:
_raise_runtime_error("Could not determine torrent hash from URL")
@@ -397,56 +382,6 @@ class RTorrentClient(DownloadClient):
except _RTORRENT_CLIENT_ERRORS:
return "/downloads"
def _list_torrent_hashes(self) -> set[str] | None:
"""Snapshot the hashes rTorrent currently reports."""
try:
all_torrents = self._rpc.d.multicall2("", "", "d.hash=")
except _RTORRENT_CLIENT_ERRORS as e:
logger.debug("Could not snapshot rTorrent downloads: %s", e)
return None
return {str(row[0]).lower() for row in all_torrents if row and row[0]}
def _discover_added_torrent_hash(
self,
name: str,
label: str,
known_hashes: set[str] | None,
) -> str | None:
"""Recover the hash of a torrent that was added without a known info_hash.
rTorrent fetches .torrent URLs itself, so the add can succeed even when
no hash could be extracted up front. A `known_hashes` of None means the
pre-add snapshot failed, so only an exact name match can identify the
new arrival.
"""
for _ in range(20):
try:
all_torrents = self._rpc.d.multicall2("", "", "d.hash=", "d.name=", "d.custom1=")
except _RTORRENT_CLIENT_ERRORS as e:
logger.debug("rTorrent hash discovery: %s", e)
else:
new_torrents = [
row
for row in all_torrents
if row
and row[0]
and (known_hashes is None or str(row[0]).lower() not in known_hashes)
]
# The label set at add time distinguishes concurrent arrivals,
# but rTorrent may not have applied it yet, so it only ever
# narrows a non-empty candidate list.
if label:
labeled = [row for row in new_torrents if len(row) > 2 and row[2] == label]
if labeled:
new_torrents = labeled
for row in new_torrents:
if len(row) > 1 and row[1] == name:
return str(row[0]).lower()
if known_hashes is not None and len(new_torrents) == 1:
return str(new_torrents[0][0]).lower()
time.sleep(0.5)
return None
def _get_torrent_path(self, download_id: str) -> str | None:
"""Get the file path of a torrent by hash.
+1 -8
View File
@@ -748,18 +748,11 @@ def prowlarr_clients_settings() -> list[SettingsField]:
TextField(
key="RTORRENT_LABEL",
label="Book Label",
description="Label to assign to ebook downloads in rTorrent",
description="Label to assign to book downloads in rTorrent",
placeholder="cwabd",
default="cwabd",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
),
TextField(
key="RTORRENT_AUDIOBOOK_LABEL",
label="Audiobook Label",
description="Label to assign to audiobook downloads in rTorrent (falls back to Book Label if not set)",
placeholder="audiobooks",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
),
TextField(
key="RTORRENT_DOWNLOAD_DIR",
label="Download Directory",
+27 -35
View File
@@ -19,7 +19,6 @@ from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_MAGNET_RESPONSE_MAX_BYTES = 2000
_TORRENT_FETCH_MAX_REDIRECTS = 5
_BASE32_BTMH_TAG_BYTES = 34
_BTIH_INFO_BYTE_HEX = 0x20
_BTIH_PREFIX_BYTE = 0x12
@@ -96,16 +95,10 @@ def extract_torrent_info(
# Not a magnet - try to fetch and parse the .torrent file
if not fetch_torrent:
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
if not _is_trusted_torrent_fetch_url(url):
logger.debug("Skipping torrent prefetch for untrusted URL: %s...", url[:80])
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
# A release source can legitimately hand us a download URL on a different
# origin than the configured Prowlarr/Newznab endpoint (e.g. a direct
# tracker link, or Prowlarr reached through a separate proxy), and a trusted
# Prowlarr download URL commonly redirects to the indexer's own download
# link. We still need to fetch the .torrent to recover the info_hash when
# the source did not provide one, so the prefetch runs regardless of origin
# and follows cross-origin redirects. The Prowlarr API key, however, is
# re-evaluated per hop and only ever sent to a trusted origin so it can
# never leak to an arbitrary indexer/tracker host.
headers: dict[str, str] = {"Accept": "application/x-bittorrent"}
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
@@ -121,28 +114,19 @@ def extract_torrent_info(
try:
logger.debug("Fetching torrent file from: %s...", url[:80])
# Redirects are followed manually: some indexers redirect download URLs
# to magnet links, and each hop must decide anew whether it may see the
# API key.
current_url = url
redirects_remaining = _TORRENT_FETCH_MAX_REDIRECTS
while True:
request_headers = dict(headers)
if not _is_trusted_torrent_fetch_url(current_url):
request_headers.pop("X-Api-Key", None)
# Use allow_redirects=False to handle magnet link redirects manually
# Some indexers redirect download URLs to magnet links
resp = requests.get(
url,
timeout=30,
allow_redirects=False,
headers=headers,
verify=get_ssl_verify(url),
)
resp = requests.get(
current_url,
timeout=30,
allow_redirects=False,
headers=request_headers,
verify=get_ssl_verify(current_url),
)
if resp.status_code not in (301, 302, 303, 307, 308):
break
redirect_url = resolve_url(current_url, resp.headers.get("Location", ""))
# Check if this is a redirect to a magnet link
if resp.status_code in (301, 302, 303, 307, 308):
redirect_url = resolve_url(url, resp.headers.get("Location", ""))
if redirect_url.startswith("magnet:"):
logger.debug("Download URL redirected to magnet link")
info_hash = extract_hash_from_magnet(redirect_url)
@@ -154,12 +138,20 @@ def extract_torrent_info(
is_magnet=True,
magnet_url=redirect_url,
)
if redirects_remaining <= 0:
logger.debug("Too many redirects fetching torrent file: %s...", url[:80])
if not _is_trusted_torrent_fetch_url(redirect_url):
logger.debug(
"Skipping torrent prefetch redirect to untrusted URL: %s...",
redirect_url[:80],
)
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
redirects_remaining -= 1
# Not a magnet redirect, follow it manually
logger.debug("Following redirect to: %s...", redirect_url[:80])
current_url = redirect_url
resp = requests.get(
redirect_url,
timeout=30,
headers=headers,
verify=get_ssl_verify(redirect_url),
)
resp.raise_for_status()
torrent_data = resp.content
+2 -6
View File
@@ -316,12 +316,8 @@ class TransmissionClient(DownloadClient):
state, message = status_map.get(status_value, ("downloading", "Downloading"))
progress = torrent.percent_done * 100
# Only mark complete when seeding or stopped (e.g. if seed limit/ratio is 0)
# and progress is complete. seed pending means files still being moved
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value in (
"seeding",
"stopped",
)
# Only mark complete when seeding - seed pending means files still being moved
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value == "seeding"
if complete:
message = "Complete"
-93
View File
@@ -852,99 +852,6 @@ def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver:
return custom_resolver
# --- ISP / network DNS interference detection ---------------------------------
# Compare what the (tamperable) system resolver returns for a host against a
# tamper-resistant DoH lookup. Divergent answers are a strong signal the network is
# hijacking or NXDOMAIN-blocking the domain (a common reason AA downloads "work" but
# land on an ISP block page). Used to surface an actionable hint to the user.
_dns_interference_warned: set[str] = set()
_dns_interference_active = False
def _build_detection_doh_resolver() -> DoHResolver | None:
"""Build a throwaway DoH resolver for interference checks (no socket patching).
Honours the DoH provider the user selected (``DNS_PROVIDERS[_current_dns_index]``),
falling back to the first configured provider when none is active. The endpoint is
pinned to the provider's own nameserver IP so resolving the DoH host can't be
redirected by the very DNS layer the check is meant to detect.
"""
if 0 <= _current_dns_index < len(DNS_PROVIDERS):
_name, servers, doh_url = DNS_PROVIDERS[_current_dns_index]
elif DNS_PROVIDERS:
_name, servers, doh_url = DNS_PROVIDERS[0]
else:
return None
server_hostname = urllib.parse.urlparse(doh_url).hostname or ""
if not server_hostname or not servers:
return None
return DoHResolver(doh_url, server_hostname, servers[0])
def detect_dns_interference(hostname: str) -> dict[str, list[str]] | None:
"""Detect network DNS interference by comparing system DNS against DoH.
Returns ``{"system_ips": [...], "doh_ips": [...]}`` when the two resolvers disagree
(no overlapping IPs), otherwise None. No-op for IP literals / local hostnames and
when DoH resolution is unavailable, so it never produces a false positive.
"""
host = (hostname or "").strip().lower()
if not host or _is_ip_address(host) or _is_local_address(host):
return None
resolver = _build_detection_doh_resolver()
if resolver is None:
return None
try:
system_ips = {str(info[4][0]) for info in original_getaddrinfo(host, 443, socket.AF_INET)}
except OSError:
return None
if not system_ips:
return None
doh_ips = {ip for ip in resolver.resolve(host, "A") if ip}
if not doh_ips or (system_ips & doh_ips):
return None
return {"system_ips": sorted(system_ips), "doh_ips": sorted(doh_ips)}
def note_possible_dns_interference(hostname: str) -> bool:
"""Check ``hostname`` for DNS interference, logging an actionable warning once.
Returns True when interference has been detected this session. The check runs at
most once per host to avoid repeated DoH lookups and log spam.
"""
global _dns_interference_active
host = (hostname or "").strip().lower()
if not host or host in _dns_interference_warned:
return _dns_interference_active
_dns_interference_warned.add(host)
result = detect_dns_interference(host)
if not result:
return _dns_interference_active
_dns_interference_active = True
routing_via_doh = _current_dns_index >= 0 and bool(DOH_SERVER)
remedy = (
"Shelfmark is routing this domain through DNS-over-HTTPS to work around it."
if routing_via_doh
else "Enable DNS-over-HTTPS (USE_DOH=true) or set a custom DNS provider to bypass it."
)
logger.warning(
"Possible ISP/network DNS interference for %s: system DNS resolves to %s but DoH "
"resolves to %s. The network appears to be blocking or redirecting this domain. %s",
host,
result["system_ips"],
result["doh_ips"],
remedy,
)
return True
def dns_interference_detected() -> bool:
"""Whether network DNS interference has been detected this session."""
return _dns_interference_active
def init_doh_resolver(doh_server: str = "") -> DoHResolver | None:
"""Initialize DNS over HTTPS resolver."""
server = doh_server or DOH_SERVER
@@ -2,7 +2,6 @@
from __future__ import annotations
import contextlib
import uuid
from typing import TYPE_CHECKING
@@ -41,11 +40,9 @@ def validate_destination(
status_callback("error", f"Destination is not a directory: {destination}")
return False
created_by_us = False
if not destination_exists:
try:
run_blocking_io(destination.mkdir, parents=True, exist_ok=True)
created_by_us = True
except (OSError, PermissionError) as exc:
log_path_permission_context("destination_create", destination)
logger.warning("Cannot create destination: %s (%s)", destination, exc)
@@ -66,9 +63,6 @@ def validate_destination(
log_path_permission_context("destination_write_probe", destination)
logger.warning("Destination not writable: %s (%s)", destination, exc)
status_callback("error", f"Destination not writable: {destination} ({exc})")
if created_by_us:
with contextlib.suppress(OSError):
run_blocking_io(destination.rmdir)
return False
return True
-3
View File
@@ -1163,9 +1163,6 @@ def api_config() -> Response | tuple[Response, int]:
"show_combined_selector": app_config.get(
"SHOW_COMBINED_SELECTOR", True, user_id=db_user_id
),
"force_combined_search": app_config.get(
"FORCE_COMBINED_SEARCH", False, user_id=db_user_id
),
"books_output_mode": app_config.get("BOOKS_OUTPUT_MODE", "folder"),
"auto_open_downloads_sidebar": app_config.get("AUTO_OPEN_DOWNLOADS_SIDEBAR", True),
"hardcover_auto_remove_on_download": app_config.get(
+1 -1
View File
@@ -2628,7 +2628,7 @@ class HardcoverProvider(MetadataProvider):
raise RuntimeError(msg) from e
return None
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.UNAUTHORIZED:
if e.response.status_code == HTTPStatus.UNAUTHORIZED:
logger.exception("Hardcover API key is invalid")
if raise_on_error:
msg = "Hardcover API key is invalid"
+3 -3
View File
@@ -214,7 +214,7 @@ class OpenLibraryProvider(MetadataProvider):
logger.warning("Open Library search timed out")
return []
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
if e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
logger.warning("Open Library service unavailable (503)")
else:
logger.exception("Open Library HTTP error")
@@ -253,7 +253,7 @@ class OpenLibraryProvider(MetadataProvider):
logger.warning("Open Library get_book timed out")
return None
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.NOT_FOUND:
if e.response.status_code == HTTPStatus.NOT_FOUND:
logger.debug("Open Library work not found: %s", book_id)
else:
logger.exception("Open Library HTTP error")
@@ -314,7 +314,7 @@ class OpenLibraryProvider(MetadataProvider):
return self._parse_edition(edition, clean_isbn)
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.NOT_FOUND:
if e.response.status_code == HTTPStatus.NOT_FOUND:
logger.debug("Open Library ISBN not found: %s", isbn)
else:
logger.exception("Open Library ISBN search HTTP error")
@@ -417,18 +417,6 @@ def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") ->
# Clean up info hash (remove whitespace, ensure uppercase)
info_hash = re.sub(r"\s+", "", info_hash).upper()
# Validate: SHA1 = 40 hex chars, SHA256 = 64 hex chars
if not re.match(r"^[0-9A-F]{40}$|^[0-9A-F]{64}$", info_hash):
logger.warning("Info Hash invalid (got %r), trying magnet fallback.", info_hash)
# Fallback: search entire page for a complete magnet link (e.g. posted in comments)
magnet_match = re.search(r"magnet:\?xt=urn:btih:([0-9a-fA-F]{40,64})", detail_html)
if magnet_match:
info_hash = magnet_match.group(1).upper()
logger.info("Found hash via magnet fallback: %s", info_hash)
else:
logger.warning("No valid magnet link found on page, giving up.")
return None
# 2. Extract Trackers
# Find all <td> containing udp:// or http://
trackers = []
@@ -238,8 +238,8 @@ class AudiobookBaySource(ReleaseSource):
exact_phrase=exact_phrase,
)
# Fallback to broad matching if exact phrase returns nothing (manual or auto query).
if exact_phrase and not results:
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
if exact_phrase and not results and not plan.manual_query:
logger.info(
"No exact phrase results, retrying AudiobookBay search without quotes"
)
@@ -288,7 +288,7 @@ class AudiobookBaySource(ReleaseSource):
size_str = result.get("size")
size_bytes = parse_size(size_str) if size_str else None
language_raw = result.get("language")
language_code = _map_language(language_raw) if language_raw else "en"
language_code = _map_language(language_raw) if language_raw else None
bitrate = result.get("bitrate")
bitrate_kbps = _parse_bitrate_to_kbps(bitrate)
+23 -320
View File
@@ -3,12 +3,9 @@
import itertools
import json
import re
import threading
import time
import unicodedata
from dataclasses import replace
from http import HTTPStatus
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, NoReturn, TypedDict
from urllib.parse import quote, urlparse
@@ -200,49 +197,6 @@ _SOURCE_FAILURE_THRESHOLD = 4
_MIN_VALID_FILE_SIZE = 10 * 1024
_AA_COUNTDOWN_MAX_SECONDS = 300
# --- Distant-path language detection ---
_DISTANT_PATH_EXTENSIONS = (
"epub",
"mobi",
"azw3",
"fb2",
"djvu",
"cbz",
"cbr",
"pdf",
"zip",
"rar",
"m4b",
"mp3",
)
_DISTANT_PATH_EXTENSION_PATTERN = "|".join(re.escape(e) for e in _DISTANT_PATH_EXTENSIONS)
_DISTANT_PATH_PATTERN = re.compile(
rf"(?:[A-Za-z0-9._-]+/)?[A-Za-z]:(?:\\|/)[^\n\r<>\"]+?\.(?:{_DISTANT_PATH_EXTENSION_PATTERN})\b",
re.IGNORECASE,
)
_DISTANT_PATH_FALLBACK_PATTERN = re.compile(
r"(?:[A-Za-z0-9._-]+/)?[A-Za-z]:(?:\\|/)[^\n\r<>\"]+",
re.IGNORECASE,
)
_BRACKETED_LANGUAGE_CODE_PATTERN = re.compile(
r"\[(?:bd[\s._-]*)?([A-Za-z]{2,3})\]",
re.IGNORECASE,
)
_KEYED_LANGUAGE_CODE_PATTERN = re.compile(
r"\b(?:bd|lang(?:uage)?)\s*[:._-]?\s*([A-Za-z]{2,3})\b",
re.IGNORECASE,
)
_LANGUAGE_CODE_TOKEN_PATTERN = re.compile(
r"(?:^|[\s_./\\\-\[(])([A-Za-z]{2,3})(?=$|[\s_./\\\-)\]])"
)
_LANGUAGE_NAME_TOKEN_PATTERN = re.compile(r"[a-z]{4,}(?:-[a-z0-9]+)?")
_LANGUAGE_ALIAS_TO_CODE: dict[str, str] | None = None
_LANGUAGE_ALIAS_LOCK = threading.Lock()
_LANGUAGE_PLACEHOLDERS = frozenset({"", "-", "--", "unknown", "unk", "n/a", "na"})
# Short codes that appear in common words — require bracket/key context to accept
_AMBIGUOUS_SHORT_LANGUAGE_CODES = frozenset({"de", "en", "it", "la", "no", "or", "is", "in"})
# Sources that require Cloudflare bypass
_CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "welib"})
@@ -250,189 +204,6 @@ _CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "weli
_AA_PAGE_SOURCES = frozenset({"aa-slow-nowait", "aa-slow-wait"})
def _is_language_from_path_enabled() -> bool:
return bool(config.get("DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH", False))
def _normalize_language_token(value: str) -> str:
normalized = value.strip().lower()
for dash in ("‑", "–", "—", "−"):
normalized = normalized.replace(dash, "-")
return normalized
def _fold_text(value: str) -> str:
normalized = unicodedata.normalize("NFKD", value)
return "".join(c for c in normalized if not unicodedata.combining(c)).lower()
def _language_alias_to_code() -> dict[str, str]:
"""Build alias→code map from bundled language metadata (lazy, cached)."""
global _LANGUAGE_ALIAS_TO_CODE
cached = _LANGUAGE_ALIAS_TO_CODE
if cached is not None:
return cached
with _LANGUAGE_ALIAS_LOCK:
cached = _LANGUAGE_ALIAS_TO_CODE
if cached is not None:
return cached
mapping: dict[str, str] = {}
data_path = Path(__file__).resolve().parents[2] / "data" / "book-languages.json"
try:
raw = json.loads(data_path.read_text(encoding="utf-8"))
except OSError, ValueError, TypeError:
_LANGUAGE_ALIAS_TO_CODE = {}
return _LANGUAGE_ALIAS_TO_CODE
if not isinstance(raw, list):
_LANGUAGE_ALIAS_TO_CODE = {}
return _LANGUAGE_ALIAS_TO_CODE
for item in raw:
if not isinstance(item, dict):
continue
code = _normalize_language_token(str(item.get("code", "")))
name = _normalize_language_token(str(item.get("language", "")))
if not code:
continue
mapping.setdefault(code, code)
mapping.setdefault(code.replace("-", "_"), code)
mapping.setdefault(code.split("-")[0], code)
mapping.setdefault(_fold_text(code), code)
if name:
mapping.setdefault(name, code)
mapping.setdefault(_fold_text(name), code)
_LANGUAGE_ALIAS_TO_CODE = mapping
return _LANGUAGE_ALIAS_TO_CODE
def _extract_distant_path(row: Tag, *, enabled: bool) -> str | None:
"""Extract the Windows-style file path from an AA search result row."""
if not enabled:
return None
def _normalize_candidate(text: str) -> str:
normalized = re.sub(r"\s*([\\/])\s*", r"\1", text)
normalized = re.sub(r":\s*([\\/])", r":\1", normalized)
return re.sub(
r"\s+\.(epub|mobi|azw3|fb2|djvu|cbz|cbr|pdf|zip|rar|m4b|mp3)\b",
r".\1",
normalized,
flags=re.IGNORECASE,
)
candidates = [row.get_text(" ", strip=True)]
for cell in row.find_all("td"):
cell_text = cell.get_text(" ", strip=True)
if cell_text:
candidates.append(cell_text)
best: str | None = None
for text in candidates:
for match in _DISTANT_PATH_PATTERN.findall(_normalize_candidate(text)):
candidate = match.strip().rstrip(".,;")
if best is None or len(candidate) > len(best):
best = candidate
if best is not None:
return best
for text in candidates:
for match in _DISTANT_PATH_FALLBACK_PATTERN.findall(_normalize_candidate(text)):
candidate = match.strip().rstrip(".,;")
if best is None or len(candidate) > len(best):
best = candidate
return best
def _detect_language_from_distant_path(path: str | None) -> str | None:
"""Infer a language code from distant-path tags such as [BD FR] or [Fr]."""
if not path:
return None
aliases = _language_alias_to_code()
if not aliases:
return None
folded_path = _fold_text(path)
strong_candidates: list[str] = []
for code in _BRACKETED_LANGUAGE_CODE_PATTERN.findall(path):
normalized = _normalize_language_token(code)
if normalized in aliases:
strong_candidates.append(aliases[normalized])
for code in _KEYED_LANGUAGE_CODE_PATTERN.findall(path):
normalized = _normalize_language_token(code)
if normalized in aliases:
strong_candidates.append(aliases[normalized])
non_ambiguous = [c for c in strong_candidates if c not in _AMBIGUOUS_SHORT_LANGUAGE_CODES]
if non_ambiguous:
return non_ambiguous[0]
for token in _LANGUAGE_NAME_TOKEN_PATTERN.findall(folded_path):
normalized = _normalize_language_token(token)
if normalized in aliases:
candidate = aliases[normalized]
if candidate not in _AMBIGUOUS_SHORT_LANGUAGE_CODES:
return candidate
if strong_candidates:
return strong_candidates[0]
for code in _LANGUAGE_CODE_TOKEN_PATTERN.findall(path):
normalized = _normalize_language_token(code)
if normalized in _AMBIGUOUS_SHORT_LANGUAGE_CODES:
continue
if normalized in aliases:
return aliases[normalized]
return None
def _is_missing_or_placeholder_language(language: str | None) -> bool:
if language is None:
return True
return _normalize_language_token(language) in _LANGUAGE_PLACEHOLDERS
def _normalize_requested_languages(languages: list[str] | None) -> set[str]:
if not languages:
return set()
aliases = _language_alias_to_code()
normalized: set[str] = set()
for value in languages:
token = _normalize_language_token(str(value))
if not token or token == "all": # noqa: S105 - "all" is a language sentinel
continue
normalized.add(aliases.get(token, token))
return normalized
def _book_matches_requested_languages(book_language: str | None, requested: set[str]) -> bool:
"""Return True when a book's language matches the requested filter.
Books with unknown/missing language always pass — the server-side &lang= filter
already narrowed the result set, so dropping unlabelled rows hides valid results.
"""
if not requested:
return True
if not book_language:
return True
aliases = _language_alias_to_code()
normalized_book = aliases.get(
_normalize_language_token(book_language),
_normalize_language_token(book_language),
)
return normalized_book in requested
def _is_configured_zlib_link(url: str) -> bool:
"""Return True when a URL belongs to a configured Z-Library mirror."""
from shelfmark.core.mirrors import get_zlib_cookie_domains
@@ -589,17 +360,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
filters_query = ""
path_language_enabled = _is_language_from_path_enabled()
requested_langs = _normalize_requested_languages(filters.lang)
# When path-language inference is on and a language is requested, skip the
# server-side &lang= filter: lgli files often have no AA language metadata
# and would be excluded before we can infer language from the distant path.
# Local filtering below handles the narrowing instead.
if not (path_language_enabled and requested_langs):
for value in filters.lang or []:
if value and value != "all":
filters_query += f"&lang={quote(value)}"
for value in filters.lang or []:
if value and value != "all":
filters_query += f"&lang={quote(value)}"
if filters.sort and filters.sort != "relevance":
filters_query += f"&sort={quote(filters.sort)}"
@@ -634,13 +397,14 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
raise SearchUnavailableError(msg)
if "No files found." in html:
logger.info("No books found for query: %s", query)
return []
soup = BeautifulSoup(_html_response_text(html), "html.parser")
tbody = soup.find("table")
if tbody is None:
if "No files found." in html:
logger.info("No books found for query: %s", query)
return []
logger.warning("No results table found for query: %s", query)
msg = "No books found. Please try another query."
raise RuntimeError(msg)
@@ -654,9 +418,6 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
if book:
books.append(book)
if path_language_enabled and requested_langs:
books = [b for b in books if _book_matches_requested_languages(b.language, requested_langs)]
supported_formats = _get_supported_formats()
books.sort(
@@ -710,23 +471,10 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
if not record_id:
return None
path_language_enabled = _is_language_from_path_enabled()
distant_path = _extract_distant_path(row, enabled=path_language_enabled)
preview_img = cells[0].find("img")
preview = _get_attr(preview_img, "src") if isinstance(preview_img, Tag) else None
title_span = cells[1].find("span")
if isinstance(title_span, Tag):
# AA nests related-edition spans inside the main title span — take only direct text.
direct = " ".join(
str(c).strip()
for c in title_span.children
if isinstance(c, NavigableString) and str(c).strip()
).strip()
title = direct or _first_stripped_text(title_span)
else:
title = None
title = _first_stripped_text(cells[1].find("span"))
author = _first_stripped_text(cells[2].find("span"))
publisher = _first_stripped_text(cells[3].find("span"))
year = _first_stripped_text(cells[4].find("span"))
@@ -735,19 +483,18 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
file_format = _first_stripped_text(cells[9].find("span"))
size = _first_stripped_text(cells[10].find("span"))
# Only title and format are truly required — lgli rows often have sparse metadata
if title is None or file_format is None:
if (
title is None
or author is None
or publisher is None
or year is None
or language is None
or content is None
or file_format is None
or size is None
):
return None
# Skip entries where the title is a catalog format descriptor, not a real title
# e.g. "Book/Online Audio", "Print book" — lgli metadata pollution
if title and "/" in title and len(title) < 40 and not any(c.isdigit() for c in title):
return None
if path_language_enabled and _is_missing_or_placeholder_language(language):
detected = _detect_language_from_distant_path(distant_path)
language = detected or "unknown"
return BrowseRecord(
id=record_id,
title=title,
@@ -760,7 +507,6 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
content=content.lower() if content else None,
format=file_format.lower() if file_format else None,
size=size,
download_path=distant_path,
)
except (AttributeError, IndexError, KeyError, TypeError) as e:
logger.error_trace(f"Error parsing search result row: {e}")
@@ -1483,9 +1229,6 @@ def _get_download_url(
return downloader.get_absolute_url(link, url)
_AA_COUNTDOWN_MAX_RETRIES = 3
def _extract_slow_download_url(
soup: BeautifulSoup,
link: str,
@@ -1494,7 +1237,6 @@ def _extract_slow_download_url(
status_callback: Callable[[str, str | None], None] | None,
selector: network.AAMirrorSelector,
source_context: str | None = None,
_countdown_attempts: int = 0,
) -> str:
"""Extract download URL from AA slow download pages."""
html_str = str(soup)
@@ -1559,14 +1301,6 @@ def _extract_slow_download_url(
countdown_seconds = _extract_countdown_seconds(soup, html_str)
if countdown_seconds > 0:
if _countdown_attempts >= _AA_COUNTDOWN_MAX_RETRIES:
logger.warning(
"Countdown retry limit (%s) reached for %s, giving up",
_AA_COUNTDOWN_MAX_RETRIES,
title,
)
return ""
max_countdown_seconds = 600
sleep_time = min(countdown_seconds, max_countdown_seconds)
if countdown_seconds > max_countdown_seconds:
@@ -1575,13 +1309,7 @@ def _extract_slow_download_url(
countdown_seconds,
max_countdown_seconds,
)
logger.info(
"AA waitlist: %ss for %s (attempt %s/%s)",
sleep_time,
title,
_countdown_attempts + 1,
_AA_COUNTDOWN_MAX_RETRIES,
)
logger.info("AA waitlist: %ss for %s", sleep_time, title)
# Live countdown with status updates
for remaining in range(sleep_time, 0, -1):
@@ -1602,31 +1330,12 @@ def _extract_slow_download_url(
if status_callback and source_context:
status_callback("resolving", f"{source_context} - Fetching")
html = downloader.html_get_page(
link, selector=selector, cancel_flag=cancel_flag, status_callback=status_callback
)
if not html:
return ""
new_soup = BeautifulSoup(_html_response_text(html), "html.parser")
return _extract_slow_download_url(
new_soup,
link,
title,
cancel_flag,
status_callback,
selector,
source_context,
_countdown_attempts + 1,
return _get_download_url(
link, title, cancel_flag, status_callback, selector, source_context
)
link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
logger.warning("No download URL found. First 10 links: %s", link_texts)
# A bypassed page with no AA download links often means the network served a wrong
# page (e.g. an ISP block page) instead of Anna's Archive. Probe for DNS interference
# so we can give the user an actionable hint instead of a generic failure.
host = urlparse(link).hostname or ""
if host:
network.note_possible_dns_interference(host)
return ""
@@ -1937,6 +1646,7 @@ class DirectDownloadSource(ReleaseSource):
except Exception:
logger.exception("Search error")
logger.info("Found %s releases via title+author", len(all_results))
return [_browse_record_to_release(record) for record in all_results]
def is_available(self) -> bool:
@@ -2059,14 +1769,7 @@ class DirectDownloadHandler(DownloadHandler):
return None
if not success_url:
if network.dns_interference_detected():
status_callback(
"error",
"All sources failed - your network/ISP appears to be blocking "
"Anna's Archive. Enable DNS-over-HTTPS in settings.",
)
else:
status_callback("error", "All download sources failed")
status_callback("error", "All download sources failed")
return None
# Return temp path - orchestrator handles post-processing (archive extraction, ingest)
+46 -21
View File
@@ -13,6 +13,7 @@ from typing import Any
from shelfmark.config import env
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.release_sources import Release, ReleaseProtocol
logger = setup_logger(__name__)
@@ -55,6 +56,12 @@ def _coerce_timestamp(value: object) -> float:
return 0.0
def _generate_cache_key(provider: str, provider_id: str, content_type: str | None = None) -> str:
"""Generate a cache key from provider, provider_id, and content type."""
normalized_content_type = "audiobook" if check_audiobook(content_type) else "ebook"
return f"{provider}:{provider_id}:{normalized_content_type}"
def _load_cache() -> dict[str, Any]:
"""Load cache from disk."""
try:
@@ -96,17 +103,17 @@ def _dict_to_release(data: dict[str, Any]) -> Release:
def get_cached_results(
cache_key: str,
provider: str,
provider_id: str,
content_type: str | None = None,
ttl_seconds: int | None = None,
) -> dict[str, Any] | None:
"""Get the cached IRC answer for a query identity (server:channel:query).
The cache stores the whole answer (releases for all content types) under the query
identity, so it is not isolated by book or content type. Callers filter by content
type after reading.
"""Get cached search results for a book.
Args:
cache_key: Query identity (e.g. "server:channel:query")
provider: Metadata provider name (e.g., "hardcover", "openlibrary")
provider_id: Book ID in the provider's system
content_type: Search content type for cache isolation
ttl_seconds: Cache TTL in seconds (from settings)
Returns:
@@ -120,6 +127,8 @@ def get_cached_results(
ttl_value = config.get("IRC_CACHE_TTL", DEFAULT_CACHE_TTL)
ttl_seconds = _coerce_cache_ttl(ttl_value, DEFAULT_CACHE_TTL)
cache_key = _generate_cache_key(provider, provider_id, content_type)
with _cache_lock:
cache = _load_cache()
entry = cache.get("entries", {}).get(cache_key)
@@ -132,9 +141,10 @@ def get_cached_results(
age = time.time() - cached_at
if ttl_seconds != 0 and age > ttl_seconds:
title = entry.get("title", cache_key)
logger.debug(
"IRC cache expired for '%s' (age: %.0fs > TTL: %ss)",
entry.get("title", cache_key),
title,
age,
ttl_seconds,
)
@@ -143,36 +153,44 @@ def get_cached_results(
# Convert dicts back to Release objects
releases = [_dict_to_release(r) for r in entry.get("releases", [])]
online_servers = entry.get("online_servers", [])
title = entry.get("title", "")
logger.info(
"IRC cache hit for '%s' (%s releases, age: %.0fs)",
entry.get("title", ""),
title,
len(releases),
age,
)
return {
"releases": releases,
"online_servers": entry.get("online_servers", []),
"online_servers": online_servers,
"cached_at": cached_at,
}
def cache_results(
cache_key: str,
provider: str,
provider_id: str,
title: str,
releases: list[Release],
content_type: str | None = None,
online_servers: list[str] | None = None,
) -> None:
"""Cache the whole IRC answer for a query identity.
"""Cache search results for a book.
Args:
cache_key: Query identity (e.g. "server:channel:query")
title: Query text (for logging/display)
releases: All Release objects from the search (every content type)
provider: Metadata provider name
provider_id: Book ID in the provider's system
title: Book title (for logging/display)
releases: List of Release objects from search
content_type: Search content type for cache isolation
online_servers: List of online server nicks (optional)
"""
cache_key = _generate_cache_key(provider, provider_id, content_type)
with _cache_lock:
cache = _load_cache()
@@ -180,6 +198,9 @@ def cache_results(
cache["entries"] = {}
cache["entries"][cache_key] = {
"provider": provider,
"provider_id": provider_id,
"content_type": "audiobook" if check_audiobook(content_type) else "ebook",
"title": title,
"releases": [_release_to_dict(r) for r in releases],
"online_servers": list(online_servers) if online_servers else [],
@@ -190,23 +211,27 @@ def cache_results(
logger.info("Cached %s IRC releases for '%s'", len(releases), title)
def invalidate_cache(cache_key: str) -> bool:
def invalidate_cache(provider: str, provider_id: str, content_type: str | None = None) -> bool:
"""Remove a specific entry from the cache.
Args:
cache_key: Query identity to remove
provider: Metadata provider name
provider_id: Book ID in the provider's system
content_type: Search content type for cache isolation
Returns:
True if entry was found and removed
"""
cache_key = _generate_cache_key(provider, provider_id, content_type)
with _cache_lock:
cache = _load_cache()
entries = cache.get("entries", {})
entry = cache.get("entries", {}).get(cache_key)
title = entry.get("title", cache_key) if entry else cache_key
if cache_key in entries:
title = entries[cache_key].get("title", cache_key)
del entries[cache_key]
if cache_key in cache.get("entries", {}):
del cache["entries"][cache_key]
_save_cache(cache)
logger.info("Invalidated IRC cache for '%s'", title)
return True
+1 -5
View File
@@ -88,11 +88,7 @@ def irc_settings() -> list[SettingsField]:
key="IRC_SEARCH_BOT",
label="Search bot",
placeholder="e.g. search",
description=(
"The search bot to address queries to (required). Searches are sent as "
'"@<bot> <query>".'
),
required=True,
description="The search bot to query for results",
env_supported=True,
),
HeadingField(
+41 -134
View File
@@ -15,7 +15,6 @@ if TYPE_CHECKING:
from shelfmark.api.websocket import ws_manager
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import is_audiobook
from shelfmark.release_sources import (
ColumnColorHint,
ColumnRenderType,
@@ -89,17 +88,6 @@ def _emit_status(message: str, phase: str = "searching") -> None:
MIN_SEARCH_INTERVAL = 15.0
_last_search_time: float = 0
# Anti-spam budget: the exact same message may only be posted to the channel a limited
# number of times within a rolling window. This stops a retry/refresh loop from flooding
# the channel with the same line over and over, while still allowing a few genuine retries
# (a search that came back empty can be tried again, and Refresh works until the budget runs
# out). Normal use never hits this: successful searches are served from the result cache
# without re-posting at all.
MAX_IDENTICAL_SENDS = 3
IDENTICAL_SEND_WINDOW_SECONDS = 24 * 60 * 60 # 24 hours
# message-send-key -> timestamps of recent posts of that exact message
_recent_message_sends: dict[str, list[float]] = {}
def _enforce_rate_limit() -> None:
"""Ensure minimum time between searches."""
@@ -114,36 +102,6 @@ def _enforce_rate_limit() -> None:
_last_search_time = time.time()
def _query_identity(server: str, channel: str, query: str) -> str:
"""Stable identity for a query on a given IRC server-channel.
Used as BOTH the result-cache key and the per-query send-counter key, so the same
query shares one cached answer and one send budget regardless of which book or
content type triggered it.
"""
return f"{server.casefold()}:{channel.casefold()}:{query.strip().casefold()}"
def _recent_send_count(key: str) -> int:
"""Number of times this exact message was posted within the rolling window."""
cutoff = time.time() - IDENTICAL_SEND_WINDOW_SECONDS
timestamps = [ts for ts in _recent_message_sends.get(key, []) if ts > cutoff]
if timestamps:
_recent_message_sends[key] = timestamps
else:
_recent_message_sends.pop(key, None)
return len(timestamps)
def _record_message_sent(key: str) -> None:
"""Record that an exact message was just posted to the channel."""
now = time.time()
cutoff = now - IDENTICAL_SEND_WINDOW_SECONDS
timestamps = [ts for ts in _recent_message_sends.get(key, []) if ts > cutoff]
timestamps.append(now)
_recent_message_sends[key] = timestamps
@register_source("irc")
class IRCReleaseSource(ReleaseSource):
"""Search IRC channels for ebook and audiobook releases."""
@@ -159,16 +117,11 @@ class IRCReleaseSource(ReleaseSource):
self._online_servers: set[str] | None = None
def is_available(self) -> bool:
"""Check if IRC is configured (server, channel, nick, and search bot are set).
The search bot is required: without it we would post bare queries straight
to the channel, which reads as spam and gets the nick banned.
"""
"""Check if IRC is configured (server, channel, and nick are set)."""
server = _config_text("IRC_SERVER")
channel = _config_text("IRC_CHANNEL")
nick = _config_text("IRC_NICK")
search_bot = _config_text("IRC_SEARCH_BOT")
return bool(server and channel and nick and search_bot)
return bool(server and channel and nick)
def get_column_config(self) -> ReleaseColumnConfig:
"""Configure UI columns for IRC results."""
@@ -226,12 +179,25 @@ class IRCReleaseSource(ReleaseSource):
logger.debug("IRC source is disabled, skipping search")
return []
# Check cache first (unless expand_search/refresh is requested)
if not expand_search:
cached = get_cached_results(book.provider, book.provider_id, content_type=content_type)
if cached:
_emit_status("Using cached results", phase="complete")
self._online_servers = set(cached.get("online_servers", []))
return cached["releases"]
# Build search query
query = plan.primary_query or self._build_query(book)
if not query:
logger.warning("No search query could be built")
return []
logger.info("IRC search: %s", query)
# Enforce rate limit
_enforce_rate_limit()
# Get IRC settings
server = _config_text("IRC_SERVER")
port = _config_port("IRC_PORT", 6697)
@@ -240,54 +206,6 @@ class IRCReleaseSource(ReleaseSource):
nick = _config_text("IRC_NICK")
search_bot = _config_text("IRC_SEARCH_BOT")
# Never post an unaddressed query to the channel. A bare book title looks like
# spam to everyone else in the channel and gets the nick banned. Searches must
# be addressed to a search bot ("@<bot> <query>").
if not search_bot:
logger.warning(
"IRC search bot not configured; refusing to post unaddressed query to channel"
)
_emit_status("IRC search bot not configured", phase="error")
return []
# One identity per query on this server-channel. The result cache and the send
# counter are both keyed on it: the SAME query shares one cached answer and one
# send budget regardless of which book/content type triggered it, while different
# queries are independent (searching 100 different books posts 100 messages).
requested = "audiobook" if is_audiobook(content_type) else "ebook"
query_key = _query_identity(server, channel, query)
# Serve the cached whole answer for an identical query (unless this is a refresh).
if not expand_search:
cached = get_cached_results(query_key)
if cached:
_emit_status("Using cached results", phase="complete")
self._online_servers = set(cached.get("online_servers", []))
return self._filter_by_content_type(cached["releases"], requested)
# Anti-spam cap: the exact same query may only be POSTED a limited number of times
# per window, even via refresh. Beyond that, serve whatever is cached rather than
# re-posting the identical message to the channel.
if _recent_send_count(query_key) >= MAX_IDENTICAL_SENDS:
logger.info(
"IRC query hit %s-send limit in window, not re-posting: %s",
MAX_IDENTICAL_SENDS,
query,
)
_emit_status(
"Search limit reached for this query — showing latest results", phase="complete"
)
cached = get_cached_results(query_key)
if cached:
self._online_servers = set(cached.get("online_servers", []))
return self._filter_by_content_type(cached["releases"], requested)
return []
logger.info("IRC search: %s", query)
# Enforce rate limit
_enforce_rate_limit()
client = None
try:
# Get or reuse IRC connection
@@ -303,27 +221,28 @@ class IRCReleaseSource(ReleaseSource):
# Capture online servers (elevated users in channel)
self._online_servers = client.online_servers
# Send search request (always addressed to the search bot, never bare)
client.send_message(f"#{channel}", f"@{search_bot} {query}")
_record_message_sent(query_key)
# Send search request
search_msg = f"@{search_bot} {query}" if search_bot else query
client.send_message(f"#{channel}", search_msg)
# Wait for results DCC - this is the long wait.
# Don't restrict the sender to the trigger bot's nick: many channels answer an
# "@search" from a differently-named results bot. The DCC endpoint/filename are
# still validated, and wait_for_dcc falls back to the channel's server list.
# Wait for results DCC - this is the long wait
_emit_status(f"Connected to #{channel} - Waiting for results...", phase="searching")
offer = client.wait_for_dcc(timeout=60.0, result_type=True)
online_servers = list(self._online_servers) if self._online_servers else None
wait_kwargs = {"expected_senders": {search_bot}} if search_bot else {}
offer = client.wait_for_dcc(timeout=60.0, result_type=True, **wait_kwargs)
if not offer:
logger.info("No search results received")
_emit_status("No results found", phase="complete")
# Release connection for reuse (don't close it)
connection_manager.release_connection(client)
# Cache the (empty) answer under the query identity so an identical query
# is served from cache instead of re-posting.
cache_results(query_key, query, [], online_servers=online_servers)
# Cache empty result to avoid repeated failed searches
cache_results(
book.provider,
book.provider_id,
book.title,
[],
content_type=content_type,
online_servers=list(self._online_servers) if self._online_servers else None,
)
return []
# Download results file
@@ -341,22 +260,19 @@ class IRCReleaseSource(ReleaseSource):
# Release connection for reuse (don't close it)
connection_manager.release_connection(client)
# A single "@search" returns one file containing every format. Parse the whole
# answer (both ebooks and audiobooks) and cache it under the query identity, so
# requesting the other content type is served from cache without re-posting.
ebook_releases = self._convert_to_releases(
parse_results_file(content, content_type="ebook"), content_type="ebook"
)
audiobook_releases = self._convert_to_releases(
parse_results_file(content, content_type="audiobook"), content_type="audiobook"
)
# Convert to Release objects
results = parse_results_file(content, content_type=content_type)
releases = self._convert_to_releases(results, content_type=content_type)
# Cache results
cache_results(
query_key,
query,
ebook_releases + audiobook_releases,
online_servers=online_servers,
book.provider,
book.provider_id,
book.title,
releases,
content_type=content_type,
online_servers=list(self._online_servers) if self._online_servers else None,
)
releases = audiobook_releases if requested == "audiobook" else ebook_releases
except DCCError as e:
logger.exception("DCC error during search")
@@ -474,15 +390,6 @@ class IRCReleaseSource(ReleaseSource):
return releases
@staticmethod
def _filter_by_content_type(releases: list[Release], requested: str) -> list[Release]:
"""Pick the requested content type out of a cached whole answer.
The cache stores releases for every content type under one query identity; each
release is tagged with its content type (defaulting to ebook when missing).
"""
return [release for release in releases if (release.content_type or "ebook") == requested]
@staticmethod
def _parse_size(size_str: str) -> int | None:
"""Parse human-readable size (e.g., '1.2MB', '500K') to bytes."""
+2 -4
View File
@@ -124,12 +124,10 @@ class ProwlarrClient:
msg = f"Invalid JSON response: {e}"
raise ValueError(msg) from e
except requests.exceptions.HTTPError as e:
status_code = e.response.status_code if e.response is not None else "unknown"
reason = e.response.reason if e.response is not None else "unknown"
logger.exception(
"Prowlarr API HTTP error: %s %s",
status_code,
reason,
e.response.status_code,
e.response.reason,
)
raise
except requests.exceptions.RequestException:
+1 -1
View File
@@ -7,5 +7,5 @@
// This script is intentionally loaded from index.html as a classic script,
// so we need to declare it as an entry point manually.
"entry": ["public/theme-init.js"],
"entry": ["public/theme-init.js"]
}
+675 -821
View File
File diff suppressed because it is too large Load Diff
+13 -13
View File
@@ -17,24 +17,24 @@
"test:unit": "vitest run"
},
"dependencies": {
"@tailwindcss/vite": "^4.3.2",
"react": "^19.2.7",
"react-dom": "^19.2.7",
"react-router-dom": "^7.18.1",
"@tailwindcss/vite": "^4.2.4",
"react": "^19.2.6",
"react-dom": "^19.2.6",
"react-router-dom": "^7.15.0",
"socket.io-client": "^4.7.5"
},
"devDependencies": {
"@types/node": "^26.0.1",
"@types/react": "^19.2.17",
"@types/node": "^25.6.2",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.3",
"knip": "^6.23.0",
"oxfmt": "^0.57.0",
"oxlint": "^1.72.0",
"oxlint-tsgolint": "^0.23.0",
"@vitejs/plugin-react": "^6.0.1",
"knip": "^6.12.1",
"oxfmt": "^0.48.0",
"oxlint": "^1.63.0",
"oxlint-tsgolint": "^0.22.1",
"tailwindcss": "^4.2.2",
"typescript": "^6.0.3",
"vite": "^8.1.0",
"vitest": "^4.1.9"
"vite": "^8.0.11",
"vitest": "^4.1.5"
}
}
+3 -10
View File
@@ -766,8 +766,7 @@ function App() {
(cfg.show_combined_selector ?? true) &&
getDefaultMode('ebook') !== 'blocked' &&
getDefaultMode('audiobook') !== 'blocked';
const nextEffectiveCombinedMode =
nextCombinedModeAllowed && (combinedMode || cfg.force_combined_search);
const nextEffectiveCombinedMode = combinedMode && nextCombinedModeAllowed;
const activeConfiguredProvider =
nextEffectiveCombinedMode && metadataProviderState.configured_provider_combined
? metadataProviderState.configured_provider_combined
@@ -865,8 +864,7 @@ function App() {
const audiobookMode = getDefaultMode('audiobook');
return ebookMode !== 'blocked' && audiobookMode !== 'blocked';
}, [effectiveSearchMode, config?.show_combined_selector, getDefaultMode]);
const combinedModeLocked = combinedModeAllowed && config?.force_combined_search === true;
const effectiveCombinedMode = combinedModeAllowed && (combinedMode || combinedModeLocked);
const effectiveCombinedMode = combinedMode && combinedModeAllowed;
const effectiveCombinedState = effectiveCombinedMode ? combinedState : null;
const defaultMetadataProviderForContentType =
@@ -1491,7 +1489,7 @@ function App() {
const handleCancel = async (id: string) => {
try {
await cancelDownload(id);
await Promise.all([fetchStatus(), refreshActivitySnapshot()]);
await fetchStatus();
} catch (error) {
console.error('Cancel failed:', error);
showToast('Failed to cancel/clear download', 'error');
@@ -2420,7 +2418,6 @@ function App() {
onContentTypeChange={setContentType}
allowedContentTypes={allowedContentTypes}
combinedMode={effectiveCombinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={combinedModeAllowed ? setCombinedMode : undefined}
queryTargets={queryTargets}
activeQueryTarget={effectiveActiveQueryTarget}
@@ -2502,7 +2499,6 @@ function App() {
onContentTypeChange={setContentType}
allowedContentTypes={allowedContentTypes}
combinedMode={effectiveCombinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={combinedModeAllowed ? setCombinedMode : undefined}
activeQueryField={activeQueryField}
searchMode={effectiveSearchMode}
@@ -2778,13 +2774,10 @@ function App() {
parsedParams={parsedParams}
config={config}
contentType={contentType}
combinedMode={combinedMode}
combinedModeAllowed={combinedModeAllowed}
advancedFilters={advancedFilters}
resolvedMetadataDefaultSort={resolvedMetadataDefaultSort}
resolvedMetadataSortOptions={resolvedMetadataSortOptions}
setContentType={setContentType}
setCombinedMode={setCombinedMode}
setSearchInput={setSearchInput}
setAdvancedFilters={setAdvancedFilters}
setShowAdvanced={setShowAdvanced}
@@ -58,8 +58,6 @@ const SEARCH_MODE_OPTIONS = [
},
];
const EMPTY_PROVIDERS: MetadataProviderSummary[] = [];
export const AdvancedFilters = ({
visible,
bookLanguages,
@@ -70,7 +68,7 @@ export const AdvancedFilters = ({
renderWrapper,
searchMode,
onSearchModeChange,
metadataProviders = EMPTY_PROVIDERS,
metadataProviders = [],
activeMetadataProvider,
onMetadataProviderChange,
contentType = 'ebook',
+3 -5
View File
@@ -1,5 +1,5 @@
import type { ReactNode } from 'react';
import { useCallback, useId, useLayoutEffect, useRef, useState } from 'react';
import { useCallback, useLayoutEffect, useRef, useState } from 'react';
import { createPortal } from 'react-dom';
import { useDismiss } from '../hooks/useDismiss';
@@ -55,7 +55,6 @@ export const Dropdown = ({
onOpenChange,
}: DropdownProps) => {
const [isOpen, setIsOpen] = useState(false);
const dropdownId = useId();
const containerRef = useRef<HTMLDivElement>(null);
const triggerRef = useRef<HTMLDivElement>(null);
const panelRef = useRef<HTMLDivElement>(null);
@@ -241,8 +240,8 @@ export const Dropdown = ({
<div className={widthClassName} ref={containerRef}>
{label && (
<label
htmlFor={dropdownId}
className="mb-1.5 block cursor-pointer text-xs font-medium text-gray-500 dark:text-gray-400"
className="mb-1.5 block text-xs font-medium text-gray-500 dark:text-gray-400"
onClick={toggleOpen}
>
{label}
</label>
@@ -252,7 +251,6 @@ export const Dropdown = ({
renderTrigger({ isOpen, toggle: toggleOpen })
) : (
<button
id={dropdownId}
type="button"
onClick={toggleOpen}
disabled={disabled}
@@ -147,7 +147,6 @@ export const DropdownList = ({
type="checkbox"
checked={selectedValues.includes(option.value)}
readOnly
aria-label={option.label}
className="pointer-events-none h-4 w-4 rounded-sm border-gray-300 text-sky-600 focus:ring-sky-500"
/>
)}
+9 -20
View File
@@ -62,7 +62,6 @@ interface HeaderProps {
onContentTypeChange?: (type: ContentType) => void;
allowedContentTypes?: ContentType[];
combinedMode?: boolean;
combinedModeLocked?: boolean;
onCombinedModeChange?: (enabled: boolean) => void;
queryTargets?: QueryTargetOption[];
activeQueryTarget?: string;
@@ -80,15 +79,6 @@ const applyTheme = (preference: string): void => {
document.documentElement.style.colorScheme = effective;
};
const DEFAULT_STATUS_COUNTS: ActivityStatusCounts = {
ongoing: 0,
completed: 0,
errored: 0,
pendingRequests: 0,
};
const EMPTY_ADMIN_USERS: ActingAsUserSelection[] = [];
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
export const Header = forwardRef<HeaderHandle, HeaderProps>(
(
{
@@ -108,7 +98,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onSettingsClick,
isAdmin = false,
canAccessSettings,
statusCounts = DEFAULT_STATUS_COUNTS,
statusCounts = { ongoing: 0, completed: 0, errored: 0, pendingRequests: 0 },
onLogoClick,
authRequired = false,
isAuthenticated = false,
@@ -116,7 +106,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
displayName,
actingAsUser = null,
onActingAsUserChange,
adminUsers = EMPTY_ADMIN_USERS,
adminUsers = [],
isAdminUsersLoading = false,
adminUsersError = null,
hasLoadedAdminUsers = false,
@@ -128,9 +118,8 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onContentTypeChange,
allowedContentTypes,
combinedMode,
combinedModeLocked,
onCombinedModeChange,
queryTargets = EMPTY_QUERY_TARGETS,
queryTargets = [],
activeQueryTarget = 'general',
onQueryTargetChange,
activeQueryField = null,
@@ -316,8 +305,8 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
// Determine if we should show icons only (both URLs configured)
const showIconsOnly = Boolean(calibreWebUrl && audiobookLibraryUrl);
// Icon buttons - reused for both states
const iconButtonsNode = (
// Icon buttons component - reused for both states
const IconButtons = () => (
<div className="flex items-center gap-2">
{/* Book Library Button */}
{calibreWebUrl && (
@@ -590,7 +579,6 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onClick={handleLogout}
className="hover-action shrink-0 rounded-full p-2 text-red-600 transition-colors dark:text-red-400"
title="Sign Out"
aria-label="Sign Out"
>
<svg
className="h-5 w-5"
@@ -686,7 +674,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
/>
))}
{iconButtonsNode}
<IconButtons />
</div>
{/* Search bar - appear second on mobile (below logo+icons), first on desktop (left side) */}
@@ -723,7 +711,6 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onContentTypeChange={onContentTypeChange}
allowedContentTypes={allowedContentTypes}
combinedMode={combinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={onCombinedModeChange}
queryTargets={queryTargets}
activeQueryTarget={activeQueryTarget}
@@ -736,7 +723,9 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
{/* When search is NOT active: show icon buttons only on the right */}
{!showSearch && (
<div className="flex min-h-[48px] items-center justify-end">{iconButtonsNode}</div>
<div className="flex min-h-[48px] items-center justify-end">
<IconButtons />
</div>
)}
</div>
</header>
@@ -101,7 +101,6 @@ const PasswordLoginForm = ({
type="text"
id="username"
name="username"
aria-label="Username"
autoComplete="username"
autoCapitalize="none"
autoCorrect="off"
@@ -132,7 +131,6 @@ const PasswordLoginForm = ({
type={showPassword ? 'text' : 'password'}
id="password"
name="password"
aria-label="Password"
autoComplete="current-password"
autoCapitalize="none"
autoCorrect="off"
@@ -170,7 +168,6 @@ const PasswordLoginForm = ({
checked={rememberMe}
onChange={(event) => setRememberMe(event.target.checked)}
disabled={isLoading}
aria-label="Remember me for 7 days"
className="h-4 w-4 rounded-sm accent-sky-900 focus:ring-2 focus:ring-sky-500 disabled:cursor-not-allowed disabled:opacity-50"
style={{ borderColor: 'var(--border-color)' }}
/>
+1 -4
View File
@@ -729,8 +729,6 @@ function ErrorState({ message }: { message: string }) {
);
}
const EMPTY_SUPPORTED_AUDIOBOOK_FORMATS: string[] = [];
const ReleaseModalSession = ({
book,
onClose,
@@ -739,7 +737,7 @@ const ReleaseModalSession = ({
onRequestBook,
getPolicyModeForSource,
supportedFormats,
supportedAudiobookFormats = EMPTY_SUPPORTED_AUDIOBOOK_FORMATS,
supportedAudiobookFormats = [],
contentType,
defaultLanguages,
bookLanguages,
@@ -2096,7 +2094,6 @@ const ReleaseModalSession = ({
value={manualQuery}
onChange={(e) => setManualQuery(e.target.value)}
placeholder="Type a custom search query (overrides all sources)"
aria-label="Custom search query"
className="w-full rounded-lg border border-(--border-muted) bg-(--bg) px-3 py-2 text-sm text-(--text)"
/>
<button
@@ -64,11 +64,9 @@ const getRequestConfirmationSessionKey = (payload: CreateRequestPayload): string
].join('|');
};
const EMPTY_PAYLOADS: CreateRequestPayload[] = [];
export function RequestConfirmationModal({
payload,
extraPayloads = EMPTY_PAYLOADS,
extraPayloads = [],
allowNotes,
onConfirm,
onClose,
@@ -91,7 +89,7 @@ export function RequestConfirmationModal({
function RequestConfirmationModalSession({
payload,
extraPayloads = EMPTY_PAYLOADS,
extraPayloads = [],
allowNotes,
onConfirm,
onClose,
@@ -307,7 +305,6 @@ function RequestConfirmationModalSession({
</label>
<textarea
id="request-note"
aria-label="Note (optional)"
value={note}
onChange={(event) => setNote(truncateRequestNote(event.target.value))}
maxLength={MAX_REQUEST_NOTE_LENGTH}
+6 -18
View File
@@ -34,7 +34,6 @@ interface SearchBarProps {
onContentTypeChange?: (type: ContentType) => void;
allowedContentTypes?: ContentType[];
combinedMode?: boolean;
combinedModeLocked?: boolean;
onCombinedModeChange?: (enabled: boolean) => void;
queryTargets?: QueryTargetOption[];
activeQueryTarget?: string;
@@ -49,7 +48,6 @@ export interface SearchBarHandle {
const EMPTY_SORT_OPTIONS: SortOption[] = [];
const EMPTY_AUTOCOMPLETE_OPTIONS: DynamicFieldOption[] = [];
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
const BookIcon = () => (
<svg
@@ -182,9 +180,8 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
onContentTypeChange,
allowedContentTypes,
combinedMode = false,
combinedModeLocked = false,
onCombinedModeChange,
queryTargets = EMPTY_QUERY_TARGETS,
queryTargets = [],
activeQueryTarget = 'general',
onQueryTargetChange,
activeQueryField,
@@ -521,7 +518,6 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
type="checkbox"
checked={Boolean(value)}
onChange={(e) => onChange(e.target.checked)}
aria-label={activeQueryField.label}
className="h-4 w-4 rounded-sm border-(--border-muted) text-emerald-500 focus:ring-emerald-500/50"
/>
<span className="truncate text-sm" style={{ color: 'var(--text)' }}>
@@ -800,19 +796,11 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
stroke="currentColor"
aria-hidden="true"
>
{combinedModeLocked ? (
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M16.5 10.5V6.75a4.5 4.5 0 1 0-9 0v3.75m-.75 11.25h10.5a2.25 2.25 0 0 0 2.25-2.25v-6.75a2.25 2.25 0 0 0-2.25-2.25H6.75a2.25 2.25 0 0 0-2.25 2.25v6.75a2.25 2.25 0 0 0 2.25 2.25Z"
/>
) : (
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
/>
)}
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
/>
</svg>
</div>
</div>
@@ -32,7 +32,6 @@ interface SearchSectionProps {
onContentTypeChange?: (type: ContentType) => void;
allowedContentTypes?: ContentType[];
combinedMode?: boolean;
combinedModeLocked?: boolean;
onCombinedModeChange?: (enabled: boolean) => void;
activeQueryField?: MetadataSearchField | null;
searchMode: SearchMode;
@@ -65,7 +64,6 @@ export const SearchSection = ({
onContentTypeChange,
allowedContentTypes,
combinedMode,
combinedModeLocked,
onCombinedModeChange,
activeQueryField,
searchMode,
@@ -107,7 +105,6 @@ export const SearchSection = ({
onContentTypeChange={onContentTypeChange}
allowedContentTypes={allowedContentTypes}
combinedMode={combinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={onCombinedModeChange}
queryTargets={queryTargets}
activeQueryTarget={activeQueryTarget}
@@ -12,13 +12,10 @@ interface UrlSearchBootstrapMountProps {
parsedParams: ParsedUrlSearch;
config: AppConfig;
contentType: ContentType;
combinedMode: boolean;
combinedModeAllowed: boolean;
advancedFilters: AdvancedFilterState;
resolvedMetadataDefaultSort: string;
resolvedMetadataSortOptions: SortOption[];
setContentType: (value: ContentType) => void;
setCombinedMode: (value: boolean) => void;
setSearchInput: (value: string) => void;
setAdvancedFilters: Dispatch<SetStateAction<AdvancedFilterState>>;
setShowAdvanced: (value: boolean) => void;
@@ -35,13 +32,10 @@ export const UrlSearchBootstrapMount = ({
parsedParams,
config,
contentType,
combinedMode,
combinedModeAllowed,
advancedFilters,
resolvedMetadataDefaultSort,
resolvedMetadataSortOptions,
setContentType,
setCombinedMode,
setSearchInput,
setAdvancedFilters,
setShowAdvanced,
@@ -55,19 +49,11 @@ export const UrlSearchBootstrapMount = ({
const parsedSearchMode = config.search_mode || 'universal';
const urlContentTypeOverride =
parsedSearchMode === 'universal' ? parsedParams.contentType : undefined;
const urlForcesCombined =
parsedSearchMode === 'universal' && parsedParams.combinedMode === true && combinedModeAllowed;
if (urlContentTypeOverride && urlContentTypeOverride !== contentType) {
setContentType(urlContentTypeOverride);
}
if (urlForcesCombined && !combinedMode) {
setCombinedMode(true);
} else if (urlContentTypeOverride && combinedMode) {
setCombinedMode(false);
}
if (!parsedParams.hasSearchParams) {
return;
}
@@ -436,7 +436,6 @@ const RejectInlinePanel = ({
Reject request for <span className="opacity-80">{itemTitle || 'Untitled request'}</span>
</p>
<textarea
aria-label="Optional note shown to the user"
value={rejectNote}
onChange={(event) => setRejectNote(event.target.value.slice(0, MAX_ADMIN_NOTE_LENGTH))}
rows={3}
@@ -225,9 +225,6 @@ const getInitialPinnedPreference = (): boolean => {
}
};
const EMPTY_KEYS: string[] = [];
const EMPTY_ITEMS: ActivityItem[] = [];
export const ActivitySidebar = ({
isOpen,
onClose,
@@ -238,8 +235,8 @@ export const ActivitySidebar = ({
onRetry,
onDownloadDismiss,
requestItems,
dismissedItemKeys = EMPTY_KEYS,
historyItems = EMPTY_ITEMS,
dismissedItemKeys = [],
historyItems = [],
historyLoaded = false,
historyHasMore = false,
historyLoading = false,
@@ -67,7 +67,6 @@ export const CardView = ({
};
return (
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
<article
className="book-card space-between animate-pop-up relative flex h-full w-full flex-col transition-shadow duration-300 max-sm:h-[180px] max-sm:flex-row sm:max-w-[292px] sm:flex-col"
style={{
@@ -70,7 +70,6 @@ export const CompactView = ({
};
return (
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
<article
className="book-card animate-pop-up relative flex! h-[180px]! w-full flex-row! transition-shadow duration-300"
style={{
@@ -102,7 +102,6 @@ export const NamingTemplateField = ({
value={value}
onChange={(event) => onChange(boundField.key, event.target.value)}
placeholder={boundField.placeholder}
aria-label={boundField.placeholder || 'Naming Template'}
maxLength={boundField.maxLength}
disabled={fieldDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
@@ -19,7 +19,6 @@ export const NumberField = ({ field, value, onChange, disabled }: NumberFieldPro
min={field.min}
max={field.max}
step={field.step ?? 1}
aria-label={field.label || 'Number field'}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
/>
@@ -21,7 +21,6 @@ export const PasswordField = ({ field, value, onChange, disabled }: PasswordFiel
value={value}
onChange={(e) => onChange(e.target.value)}
placeholder={field.placeholder}
aria-label={field.label || field.placeholder || 'Password field'}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 pr-10 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
/>
@@ -277,7 +277,6 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
checked={Boolean(cellValue)}
onChange={(e) => updateCell(rowIndex, col.key, e.target.checked)}
disabled={isDisabled}
aria-label={`${col.label || col.key} row ${rowIndex + 1}`}
className="h-4 w-4 rounded border-gray-300 text-sky-600 focus:ring-sky-500 disabled:cursor-not-allowed disabled:opacity-60"
/>
</div>
@@ -362,7 +361,6 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
value={toPrimitiveString(cellValue)}
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
placeholder={col.placeholder}
aria-label={`${col.label || col.key} row ${rowIndex + 1}`}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
/>
@@ -143,7 +143,6 @@ export const TagListField = ({
type="text"
value={draft}
onChange={(e) => setDraft(e.target.value)}
aria-label={field.label || field.placeholder || 'Add item'}
onKeyDown={(e) => {
if (e.key === 'Enter') {
e.preventDefault();
@@ -17,7 +17,6 @@ export const TextField = ({ field, value, onChange, disabled }: TextFieldProps)
value={value ?? ''}
onChange={(e) => onChange(e.target.value)}
placeholder={field.placeholder}
aria-label={field.label || field.placeholder || 'Text field'}
maxLength={field.maxLength}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
@@ -139,11 +139,9 @@ function formatUserOverrideValue(value: unknown): string {
}
}
const EMPTY_DETAILS: Array<{ userId: number; username: string; value: unknown }> = [];
const UserOverriddenBadge = ({
count,
details = EMPTY_DETAILS,
details = [],
}: {
count: number;
details?: Array<{ userId: number; username: string; value: unknown }>;
@@ -60,8 +60,6 @@ const modeDescriptions: Record<RequestPolicyMode, string> = {
blocked: 'Downloads and requests are blocked.',
};
const EMPTY_BASE_RULES: RequestPolicyRuleRow[] = [];
export const RequestPolicyGrid = ({
defaultModes,
onDefaultModeChange,
@@ -69,7 +67,7 @@ export const RequestPolicyGrid = ({
defaultModeOverrides,
defaultModeDisabled,
explicitRules,
baseRules = EMPTY_BASE_RULES,
baseRules = [],
onExplicitRulesChange,
sourceCapabilities,
rulesDisabled = false,
@@ -3,7 +3,6 @@ interface ToggleSwitchProps {
onChange: (checked: boolean) => void;
disabled?: boolean;
color?: 'sky' | 'emerald';
ariaLabel?: string;
}
const colorClasses = {
@@ -16,7 +15,6 @@ export const ToggleSwitch = ({
onChange,
disabled = false,
color = 'sky',
ariaLabel = 'Toggle switch',
}: ToggleSwitchProps) => {
const { active, ring } = colorClasses[color];
@@ -25,7 +23,6 @@ export const ToggleSwitch = ({
type="button"
role="switch"
aria-checked={checked}
aria-label={ariaLabel}
onClick={() => !disabled && onChange(!checked)}
disabled={disabled}
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors duration-200 focus:ring-2 focus:outline-hidden ${ring} disabled:cursor-not-allowed disabled:opacity-60 ${checked ? active : 'bg-gray-300 dark:bg-gray-600'}`}
@@ -188,7 +188,6 @@ export function Tooltip({
isVisible &&
coords &&
createPortal(
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- interactive tooltip stays open while hovered; keyboard users are served by focus handlers on the trigger element
<div
ref={tooltipRef}
role="tooltip"
+1 -1
View File
@@ -41,7 +41,7 @@ export function useUrlSearch({ enabled }: UseUrlSearchOptions): UseUrlSearchRetu
}
const parsed = parseUrlSearchParams(searchParams);
return parsed.hasSearchParams || parsed.contentType || parsed.combinedMode ? parsed : null;
return parsed.hasSearchParams || parsed.contentType ? parsed : null;
}, [enabled, searchParams]);
return {
@@ -42,22 +42,4 @@ describe('parseUrlSearchParams', () => {
expect(parsed.hasSearchParams).toBe(false);
expect(parsed.contentType).toBe('ebook');
});
it('parses content_type=combined as a combined-mode override', () => {
const parsed = parseUrlSearchParams(new URLSearchParams('q=dune&content_type=combined'));
expect(parsed.searchInput).toBe('dune');
expect(parsed.hasSearchParams).toBe(true);
expect(parsed.contentType).toBe(undefined);
expect(parsed.combinedMode).toBe(true);
});
it('keeps combined-only links from auto-triggering a blank search', () => {
const parsed = parseUrlSearchParams(new URLSearchParams('content_type=combined'));
expect(parsed.searchInput).toBe('');
expect(parsed.hasSearchParams).toBe(false);
expect(parsed.contentType).toBe(undefined);
expect(parsed.combinedMode).toBe(true);
});
});
-1
View File
@@ -280,7 +280,6 @@ export interface AppConfig {
default_release_source_audiobook?: string; // Default tab in ReleaseModal for audiobooks
show_release_source_links: boolean;
show_combined_selector: boolean;
force_combined_search: boolean;
books_output_mode: BooksOutputMode;
auto_open_downloads_sidebar: boolean; // Auto-open sidebar when download is queued
hardcover_auto_remove_on_download: boolean; // Auto-remove from active Hardcover list on download
+7 -15
View File
@@ -7,24 +7,18 @@ export interface ParsedUrlSearch {
searchInput: string;
advancedFilters: Partial<AdvancedFilterState>;
contentType?: ContentType;
combinedMode?: boolean;
hasSearchParams: boolean;
}
const parseContentTypeParam = (
value: string | null,
): { contentType?: ContentType; combinedMode?: true } => {
const parseContentType = (value: string | null): ContentType | undefined => {
if (!value) {
return {};
return undefined;
}
const normalized = value.trim().toLowerCase();
if (normalized === 'ebook' || normalized === 'audiobook') {
return { contentType: normalized };
return normalized;
}
if (normalized === 'combined') {
return { combinedMode: true };
}
return {};
return undefined;
};
/**
@@ -32,23 +26,21 @@ const parseContentTypeParam = (
*
* Supports both Direct Download and Universal mode parameters.
* In Universal mode, query/sort are used for search text, and content_type
* selects ebook, audiobook, or combined (search both at once).
* is used to select ebook vs audiobook.
*
* @example
* // Direct mode: /?q=harry+potter&author=rowling&format=epub&lang=en
* // Universal mode: /?q=dune&sort=popularity
* // Universal combined: /?q=dune&content_type=combined
*/
export function parseUrlSearchParams(searchParams: URLSearchParams): ParsedUrlSearch {
const contentTypeParam = parseContentTypeParam(
const parsedContentType = parseContentType(
searchParams.get('content_type') || searchParams.get('contentType'),
);
const result: ParsedUrlSearch = {
searchInput: '',
advancedFilters: {},
contentType: contentTypeParam.contentType,
combinedMode: contentTypeParam.combinedMode,
contentType: parsedContentType,
hasSearchParams: false,
};
+5 -5
View File
@@ -58,7 +58,7 @@ SAMPLE_DETAIL_HTML = """
<table>
<tr>
<td>Info Hash</td>
<td>ABC123DEF456789012345678901234567890ABCD</td>
<td>ABC123DEF456GHI789JKL012MNO345PQR678STU</td>
</tr>
<tr>
<td>Tracker 1</td>
@@ -83,7 +83,7 @@ DETAIL_HTML_NO_TRACKERS = """
<table>
<tr>
<td>Info Hash</td>
<td>ABC123DEF456789012345678901234567890ABCD</td>
<td>ABC123DEF456GHI789JKL012MNO345PQR678STU</td>
</tr>
</table>
</body>
@@ -360,7 +360,7 @@ class TestExtractMagnetLink:
assert magnet_link is not None
assert magnet_link.startswith("magnet:?xt=urn:btih:")
assert "ABC123DEF456789012345678901234567890ABCD" in magnet_link
assert "ABC123DEF456GHI789JKL012MNO345PQR678STU" in magnet_link
assert "udp%3A//tracker.openbittorrent.com%3A80" in magnet_link
assert "http%3A//tracker.example.com%3A8080" in magnet_link
assert mock_html_get.call_count == 2
@@ -395,7 +395,7 @@ class TestExtractMagnetLink:
assert magnet_link is not None
assert magnet_link.startswith("magnet:?xt=urn:btih:")
assert "ABC123DEF456789012345678901234567890ABCD" in magnet_link
assert "ABC123DEF456GHI789JKL012MNO345PQR678STU" in magnet_link
# Should contain default trackers
assert "udp%3A//tracker.openbittorrent.com%3A80" in magnet_link
@@ -441,7 +441,7 @@ class TestExtractMagnetLink:
<table>
<tr>
<td>Info Hash</td>
<td>ABC 123 DEF 456 789 012 345 678 901 234 567 890 ABC D</td>
<td>ABC 123 DEF 456</td>
</tr>
</table>
</body>
-103
View File
@@ -1,103 +0,0 @@
"""Cluster 7 (audiobook/ABB) parse-contract guards.
ABB forces ``https://`` for search and detail fetches, so it can't be exercised
hermetically in the HTTP e2e docker platform. Its recurring bugs are instead in
*parsing*: magnet/info-hash extraction ("Fix ABB magnet parsing", and the
qbittorrent hash-length issue #386) and DOM/layout drift. These contract tests
feed golden HTML through the real scraper — the same fail-on-drift philosophy as
the AA layout-drift guard — and run in normal CI.
They deliberately cover cases the existing ``test_scraper.py`` does not: info-hash
*normalization* (whitespace/case), the in-page magnet *fallback*, and a layout
drift that must degrade to an empty result rather than crash.
"""
from __future__ import annotations
import re
from unittest.mock import patch
from shelfmark.release_sources.audiobookbay import scraper
# Detail page where the Info Hash is lowercase and split by whitespace/newlines —
# the exact shape that produced malformed magnets / wrong hash lengths (#386).
DETAIL_HTML_MESSY_HASH = """
<html><body><table>
<tr><td>Info Hash</td><td>abc123def456789012345678
901234567890abcd</td></tr>
<tr><td>Tracker 1</td><td>udp://tracker.openbittorrent.com:80</td></tr>
</table></body></html>
"""
# Info Hash cell is junk, but a full magnet link is posted elsewhere on the page.
DETAIL_HTML_MAGNET_FALLBACK = """
<html><body>
<table><tr><td>Info Hash</td><td>n/a</td></tr></table>
<p>Mirror: magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=x</p>
</body></html>
"""
# DOM drift: results are present but the .post / .postTitle structure changed.
SEARCH_HTML_LAYOUT_DRIFT = """
<html><body>
<article class="result-card">
<header><a href="/abss/drifted/">Drifted Audiobook - Author</a></header>
<span class="lang">English</span>
</article>
</body></html>
"""
def _patch_detail(html: str):
return patch(
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
return_value=html,
)
def test_info_hash_is_normalized_to_canonical_btih() -> None:
"""Whitespace/newlines are stripped and the hash upper-cased to a valid
40-char btih (regression for #386 / 'Fix ABB magnet parsing')."""
with _patch_detail(DETAIL_HTML_MESSY_HASH):
magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/x/", "audiobookbay.lu")
assert magnet is not None, "messy-but-valid info hash should still yield a magnet"
btih = re.search(r"xt=urn:btih:([0-9A-Fa-f]+)", magnet)
assert btih is not None, magnet
assert btih.group(1) == "ABC123DEF456789012345678901234567890ABCD"
assert len(btih.group(1)) == 40
assert "tr=" in magnet # tracker carried through
def test_magnet_fallback_when_info_hash_cell_is_junk() -> None:
"""When the Info Hash cell is invalid, the scraper recovers the hash from an
in-page magnet link rather than failing."""
with _patch_detail(DETAIL_HTML_MAGNET_FALLBACK):
magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/y/", "audiobookbay.lu")
assert magnet is not None
assert "btih:1111111111111111111111111111111111111111" in magnet
def test_missing_info_hash_returns_none_not_crash() -> None:
"""No hash anywhere -> None (clean failure), never an exception."""
with _patch_detail("<html><body><p>nothing here</p></body></html>"):
assert (
scraper.extract_magnet_link("https://audiobookbay.lu/abss/z/", "audiobookbay.lu")
is None
)
def test_search_layout_drift_degrades_to_empty() -> None:
"""A changed results DOM yields zero parsed results without raising — the
ABB analogue of the AA layout-drift guard."""
with (
patch(
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
return_value=(SEARCH_HTML_LAYOUT_DRIFT, "https://audiobookbay.lu/?s=test"),
),
patch(
"shelfmark.release_sources.audiobookbay.scraper.config.get",
return_value=0.0,
),
):
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
assert results == [], f"drifted DOM should parse to no results, got {results!r}"
-69
View File
@@ -274,75 +274,6 @@ def test_run_child_process_writes_failure_for_unexpected_exception(monkeypatch,
assert "plain SeleniumBase startup failure" in result["traceback"]
def test_run_child_process_applies_parent_dns_config(monkeypatch, tmp_path):
"""Regression test for issue #1028: the helper subprocess must mirror the parent's
DNS provider, otherwise it pre-resolves AA hostnames against (possibly hijacked)
system DNS and Chrome loads the wrong page."""
import io
import json
import shelfmark.bypass.internal_bypasser as internal_bypasser
result_path = tmp_path / "result.json"
request = {
"url": "https://annas-archive.pk/slow_download/abc/0/0",
"retry": 1,
"result_path": str(result_path),
"dns_config": {
"provider": "cloudflare",
"servers": ["1.1.1.1", "1.0.0.1"],
"doh_url": "https://cloudflare-dns.com/dns-query",
"doh_enabled": True,
"is_auto_mode": True,
},
}
applied: list[tuple] = []
monkeypatch.setattr(
internal_bypasser.network,
"set_dns_provider",
lambda provider, manual=None, *, use_doh=None: applied.append((provider, manual, use_doh)),
)
monkeypatch.setattr(internal_bypasser, "get", lambda *_a, **_k: "<html>ok</html>")
monkeypatch.setattr(internal_bypasser.sys, "stdin", io.StringIO(json.dumps(request)))
assert internal_bypasser._run_child_process() == 0
assert applied == [("cloudflare", None, True)]
def test_apply_parent_dns_config_skips_auto_and_empty(monkeypatch):
import shelfmark.bypass.internal_bypasser as internal_bypasser
calls: list = []
monkeypatch.setattr(
internal_bypasser.network,
"set_dns_provider",
lambda *a, **k: calls.append((a, k)),
)
internal_bypasser._apply_parent_dns_config({"provider": "auto"})
internal_bypasser._apply_parent_dns_config({})
assert calls == []
def test_apply_parent_dns_config_forwards_manual_servers(monkeypatch):
import shelfmark.bypass.internal_bypasser as internal_bypasser
calls: list = []
monkeypatch.setattr(
internal_bypasser.network,
"set_dns_provider",
lambda provider, manual=None, *, use_doh=None: calls.append((provider, manual, use_doh)),
)
internal_bypasser._apply_parent_dns_config(
{"provider": "manual", "servers": ["9.9.9.9"], "doh_enabled": False}
)
assert calls == [("manual", ["9.9.9.9"], False)]
def test_prepare_child_browser_env_uses_writable_runtime_paths(monkeypatch, tmp_path):
import stat
-14
View File
@@ -368,20 +368,6 @@ def test_download_source_settings_include_direct_download_toggle():
assert "Add your own mirror URLs" in toggle_field.description
def test_download_source_settings_include_distant_path_language_toggle():
from shelfmark.config.settings import download_source_settings
fields = download_source_settings()
toggle_field = next(
field
for field in fields
if getattr(field, "key", None) == "DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH"
)
assert toggle_field.default is False
assert "distant path" in toggle_field.description.lower()
def test_fast_source_options_lock_entries_without_mirror_or_donator_requirements(monkeypatch):
from shelfmark.config.settings import _get_fast_source_options
-116
View File
@@ -257,119 +257,3 @@ def test_entrypoint_root_bootstrap_fails_closed_when_config_repair_fails(tmp_pat
assert f"Configured runtime identity: {os.getuid()}:{os.getgid()}" in result.stdout
assert f"chown -R {os.getuid()}:{os.getgid()} /path/to/config" in result.stdout
assert "Startup mode: root" not in result.stdout
def test_entrypoint_rejects_wireguard_in_non_root_mode(tmp_path):
result, _, _, _ = _run_entrypoint(tmp_path, extra_env={"USING_WIREGUARD": "true"})
assert result.returncode == 1
assert "USING_WIREGUARD=true requires the container to start as root." in result.stderr
assert "Non-root mode skips the privileged network setup WireGuard depends on." in result.stderr
def test_entrypoint_rejects_tor_and_wireguard_together(tmp_path):
result, _, _, _ = _run_entrypoint(
tmp_path,
extra_env={"USING_TOR": "true", "USING_WIREGUARD": "true"},
)
assert result.returncode == 1
assert (
"USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode."
in result.stderr
)
# The mutual-exclusion check must fire before either egress script runs, so
# neither the Tor nor the WireGuard privileged-setup errors should appear.
assert "requires the container to start as root" not in result.stderr
def test_entrypoint_mutual_exclusion_precedes_tor_startup(tmp_path):
# Even in root mode, enabling both must fail fast on mutual exclusion rather
# than starting tor.sh and then aborting.
result, _, _, _ = _run_entrypoint(
tmp_path,
simulate_root_startup=True,
extra_env={"USING_TOR": "true", "USING_WIREGUARD": "true"},
)
assert result.returncode == 1
assert (
"USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode."
in result.stderr
)
def test_entrypoint_aborts_before_gunicorn_when_wireguard_fails(tmp_path):
"""Security invariant: if wireguard.sh exits non-zero (any fail-closed path),
entrypoint.sh must abort under `set -e` so gunicorn NEVER starts. A booting
app after a failed egress setup would be a kill-switch bypass / IP leak.
"""
# Run the REAL entrypoint from a temp cwd that provides a stub `./wireguard.sh`
# which exits 1, plus a stub `./tor.sh` (unused here) for completeness.
work = tmp_path / "work"
work.mkdir()
real_entrypoint = ENTRYPOINT_PATH.read_text()
(work / "entrypoint.sh").write_text(real_entrypoint)
(work / "entrypoint.sh").chmod(0o755)
_write_executable(
work / "wireguard.sh",
"#!/bin/sh\necho 'stub wireguard.sh failing closed' >&2\nexit 1\n",
)
_write_executable(work / "tor.sh", "#!/bin/sh\nexit 0\n")
bin_dir, runtime_home_file, runtime_args_file = _build_stub_bin(tmp_path)
config_dir = tmp_path / "config"
config_dir.mkdir(exist_ok=True)
env = os.environ.copy()
env.update(
{
"BUILD_VERSION": "test-build",
"CONFIG_DIR": str(config_dir),
"DEBUG": "false",
"ENABLE_LOGGING": "false",
"ENTRYPOINT_GUNICORN_ARGS_FILE": str(runtime_args_file),
"ENTRYPOINT_GUNICORN_HOME_FILE": str(runtime_home_file),
"ENTRYPOINT_REAL_ID": ID_PATH,
"ENTRYPOINT_REAL_MKDIR": MKDIR_PATH,
"ENTRYPOINT_REAL_STAT": STAT_PATH,
"ENTRYPOINT_STUB_GID": str(os.getgid()),
"ENTRYPOINT_STUB_HOME": str(tmp_path / "runtime-home"),
"ENTRYPOINT_STUB_UID": str(os.getuid()),
# Root startup so the WireGuard branch runs ./wireguard.sh (our stub).
"ENTRYPOINT_STUB_CURRENT_GID": "0",
"ENTRYPOINT_STUB_CURRENT_UID": "0",
"ENTRYPOINT_STUB_STAT_OWNER": "0:0",
"FLASK_PORT": "8084",
"LOG_LEVEL": "info",
"LOG_ROOT": str(tmp_path / "logs"),
"PATH": f"{bin_dir}:{env.get('PATH', '')}",
"PGID": str(os.getgid()),
"PUID": str(os.getuid()),
"RELEASE_VERSION": "test-release",
"TMP_DIR": str(tmp_path / "tmp"),
"TZ": "",
"USING_EXTERNAL_BYPASSER": "true",
"USING_WIREGUARD": "true",
}
)
with _entrypoint_lock():
result = subprocess.run(
[BASH_PATH, str(work / "entrypoint.sh")],
capture_output=True,
cwd=work,
env=env,
text=True,
check=False,
)
# Entrypoint must have aborted with the stub's non-zero status...
assert result.returncode != 0
# ...the failure must actually come from wireguard.sh (not some unrelated
# earlier abort), proven by the stub's marker on stderr...
assert "stub wireguard.sh failing closed" in result.stderr
# ...and gunicorn must NEVER have been invoked (args file never written).
assert not runtime_args_file.exists(), (
"gunicorn was started despite wireguard.sh failing — kill-switch bypass!"
)
-9
View File
@@ -507,15 +507,6 @@ class TestConcurrencyConfiguration:
assert interval >= 1
assert interval <= 10
def test_completed_path_timeout_default(self):
"""Completed external-client path wait should default to the legacy grace period."""
from shelfmark.core.config import config
config.refresh()
timeout = config.get("DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT", 60)
assert timeout == 60
# =============================================================================
# Cache Configuration Tests
-1
View File
@@ -1267,7 +1267,6 @@ class TestAdminSearchPreferences:
assert data["keys"] == [
"SEARCH_MODE",
"SHOW_COMBINED_SELECTOR",
"FORCE_COMBINED_SEARCH",
"METADATA_PROVIDER",
"METADATA_PROVIDER_AUDIOBOOK",
"METADATA_PROVIDER_COMBINED",
-69
View File
@@ -597,72 +597,3 @@ def test_resolve_user_routes_expands_multiselect_event_rows(monkeypatch):
{"event": "request_fulfilled", "url": "ntfys://ntfy.sh/user-main"},
{"event": "all", "url": "ntfys://ntfy.sh/user-all"},
]
class TestAppriseProxyEnv:
"""Regression tests for issue #956 — proxy settings ignored for notifications."""
def _patch_config(self, monkeypatch, values):
from shelfmark.core import config as config_module
def _fake_get(key, default="", **_kwargs):
return values.get(key, default)
monkeypatch.setattr(config_module.config, "get", _fake_get)
def test_http_proxy_mode_injects_proxy_env(self, monkeypatch):
self._patch_config(
monkeypatch,
{
"PROXY_MODE": "http",
"HTTP_PROXY": "http://proxy.example.com:8080",
"HTTPS_PROXY": "",
"NO_PROXY": "",
},
)
monkeypatch.delenv("HTTP_PROXY", raising=False)
monkeypatch.delenv("HTTPS_PROXY", raising=False)
result = notifications_module._apprise_proxy_env()
assert result["HTTP_PROXY"] == "http://proxy.example.com:8080"
assert result["HTTPS_PROXY"] == "http://proxy.example.com:8080"
def test_socks5_proxy_mode_injects_socks_env(self, monkeypatch):
self._patch_config(
monkeypatch,
{
"PROXY_MODE": "socks5",
"SOCKS5_PROXY": "socks5://proxy.example.com:1080",
"NO_PROXY": "",
},
)
monkeypatch.delenv("HTTP_PROXY", raising=False)
monkeypatch.delenv("HTTPS_PROXY", raising=False)
result = notifications_module._apprise_proxy_env()
assert result["HTTP_PROXY"] == "socks5://proxy.example.com:1080"
assert result["HTTPS_PROXY"] == "socks5://proxy.example.com:1080"
def test_no_proxy_mode_returns_empty_dict(self, monkeypatch):
self._patch_config(monkeypatch, {"PROXY_MODE": ""})
result = notifications_module._apprise_proxy_env()
assert result == {}
def test_does_not_override_already_set_env_vars(self, monkeypatch):
self._patch_config(
monkeypatch,
{
"PROXY_MODE": "http",
"HTTP_PROXY": "http://new-proxy.example.com:8080",
"NO_PROXY": "",
},
)
monkeypatch.setenv("HTTP_PROXY", "http://existing-proxy.example.com:3128")
result = notifications_module._apprise_proxy_env()
assert "HTTP_PROXY" not in result
-25
View File
@@ -5,31 +5,6 @@ import types
import xmlrpc.client as stdlib_xmlrpc_client
from shelfmark.core import utils
from shelfmark.core.utils import normalize_http_url
class TestNormalizeHttpUrlQueryStripping:
"""Regression tests for issue #999 — mirror URLs with query params/fragments."""
def test_strips_query_string_from_configured_url(self) -> None:
result = normalize_http_url("http://mirror.example.com/search?token=abc123")
assert result == "http://mirror.example.com/search"
def test_strips_fragment_from_configured_url(self) -> None:
result = normalize_http_url("http://mirror.example.com/search#section")
assert result == "http://mirror.example.com/search"
def test_strips_both_query_and_fragment(self) -> None:
result = normalize_http_url("https://mirror.example.com/path?key=val&x=1#top")
assert result == "https://mirror.example.com/path"
def test_plain_url_unchanged(self) -> None:
result = normalize_http_url("http://mirror.example.com/search")
assert result == "http://mirror.example.com/search"
def test_trailing_slash_still_stripped_after_query_removal(self) -> None:
result = normalize_http_url("http://mirror.example.com/?token=x")
assert result == "http://mirror.example.com"
def test_get_hardened_xmlrpc_client_tolerates_patch_runtime_error(monkeypatch) -> None:
@@ -182,170 +182,3 @@ class TestDirectDownloadSearchQueries:
("mistborn custom query", ["en"], ["epub"]),
("mistborn custom query", None, ["epub"]),
]
# --- Distant-path language detection tests ---
def _patch_path_language(monkeypatch, enabled: bool = True):
import shelfmark.release_sources.direct_download as dd
original_get = dd.config.get
def _fake_get(key: str, default=None, user_id=None):
del user_id
if key == "DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH":
return enabled
return original_get(key, default)
monkeypatch.setattr(dd.config, "get", _fake_get)
return dd
def _row_from_html(html: str):
from bs4 import BeautifulSoup
return BeautifulSoup(html, "html.parser").find("tr")
def _make_row(distant_path: str, language: str = "", record_id: str = "rec-1") -> str:
return rf"""
<tr>
<td><a href="/md5/{record_id}"><img src="cover.jpg"></a></td>
<td><span>A Book Title</span></td>
<td><span>Author Name</span></td>
<td><span>Publisher</span></td>
<td><span>2024</span></td>
<td><span>-</span></td>
<td><span>-</span></td>
<td><span>{language}</span></td>
<td><span>fiction</span></td>
<td><span>epub</span></td>
<td><span>1 mb</span></td>
<td><span>{distant_path}</span></td>
</tr>
"""
def test_detects_bracketed_language_from_distant_path(monkeypatch):
dd = _patch_path_language(monkeypatch)
row = _row_from_html(_make_row(r"lgli/N:\comics1\emule\2021.08.01\[BD FR] Scrameustache.cbz"))
record = dd._parse_search_result_row(row)
assert record is not None
assert record.language == "fr"
assert record.download_path is not None
def test_detects_mixed_case_bracketed_language(monkeypatch):
dd = _patch_path_language(monkeypatch)
row = _row_from_html(_make_row(r"lgli/V:\comics\_0DAY3\[Fr]\BDs [Fr]\!Pdf\S\Book.pdf"))
record = dd._parse_search_result_row(row)
assert record is not None
assert record.language == "fr"
def test_overrides_unknown_language_with_path_detection(monkeypatch):
dd = _patch_path_language(monkeypatch)
row = _row_from_html(_make_row(r"lgli/V:\comics\_0DAY3\[Fr]\Book.pdf", language="unknown"))
record = dd._parse_search_result_row(row)
assert record is not None
assert record.language == "fr"
def test_sets_unknown_when_path_has_no_language(monkeypatch):
dd = _patch_path_language(monkeypatch)
row = _row_from_html(_make_row(r"lgli/N:\comics1\emule\NoLanguageHere.epub"))
record = dd._parse_search_result_row(row)
assert record is not None
assert record.language == "unknown"
def test_avoids_en_false_positive_when_french_present(monkeypatch):
dd = _patch_path_language(monkeypatch)
row = _row_from_html(
_make_row(r"lgli/V:\comics\_0DAY2\Stripboeken Frans - BD en Français\[BD Fr] Book.cbr")
)
record = dd._parse_search_result_row(row)
assert record is not None
assert record.language == "fr"
def test_keeps_row_with_missing_language_when_toggle_disabled(monkeypatch):
dd = _patch_path_language(monkeypatch, enabled=False)
row = _row_from_html(_make_row(r"lgli/N:\comics1\[BD FR] Scrameustache.cbz"))
record = dd._parse_search_result_row(row)
assert record is not None
assert record.language is None
def test_keeps_sparse_lgli_row(monkeypatch):
"""lgli rows missing author/publisher/year must not be dropped."""
dd = _patch_path_language(monkeypatch)
html = r"""
<tr>
<td><a href="/md5/sparse-1"><img src="cover.jpg"></a></td>
<td><span>Gos - 1978 - Le scrameustache T06.cbz</span></td>
<td></td><td></td><td></td><td></td><td></td><td></td>
<td><span>Comic book</span></td>
<td><span>cbz</span></td>
<td><span>17.4MB</span></td>
<td><span>lgli/N:\comics1\ftp\[BD.FR] French Comics\Book.cbz</span></td>
</tr>
"""
record = dd._parse_search_result_row(_row_from_html(html))
assert record is not None
assert record.id == "sparse-1"
assert record.language == "fr"
assert record.author is None
def test_search_books_filters_locally_when_path_language_enabled(monkeypatch):
dd = _patch_path_language(monkeypatch)
monkeypatch.setattr(dd.network, "get_aa_base_url", lambda: "https://mirror.example")
monkeypatch.setattr(dd.network, "AAMirrorSelector", lambda: object())
captured_url: dict[str, str] = {}
def _fake_html_get_page(url: str, selector, allow_bypasser_fallback=False):
del selector, allow_bypasser_fallback
captured_url["url"] = url
return r"""
<table>
<tr>
<td><a href="/md5/rec-fr"><img src="c.jpg"></a></td>
<td><span>Livre FR</span></td><td><span>Auteur</span></td>
<td><span>Editeur</span></td><td><span>2025</span></td>
<td><span>-</span></td><td><span>-</span></td><td></td>
<td><span>fiction</span></td><td><span>pdf</span></td>
<td><span>2 mb</span></td>
<td><span>lgli/V:\comics\_0DAY3\[Fr]\Book FR.pdf</span></td>
</tr>
<tr>
<td><a href="/md5/rec-en"><img src="c.jpg"></a></td>
<td><span>Book EN</span></td><td><span>Author</span></td>
<td><span>Publisher</span></td><td><span>2025</span></td>
<td><span>-</span></td><td><span>-</span></td><td></td>
<td><span>fiction</span></td><td><span>pdf</span></td>
<td><span>2 mb</span></td>
<td><span>lgli/V:\comics\_0DAY3\[En]\Book EN.pdf</span></td>
</tr>
</table>
"""
monkeypatch.setattr(dd.downloader, "html_get_page", _fake_html_get_page)
records = dd.search_books("demo", SearchFilters(lang=["fr"], format=["pdf"]))
assert "&lang=" not in captured_url["url"]
assert len(records) == 1
assert records[0].id == "rec-fr"
assert records[0].language == "fr"
def test_book_matches_requested_languages_logic():
import shelfmark.release_sources.direct_download as dd
assert dd._book_matches_requested_languages(None, {"fr"}) is True
assert dd._book_matches_requested_languages(None, set()) is True
assert dd._book_matches_requested_languages("en", {"fr"}) is False
assert dd._book_matches_requested_languages("fr", {"fr"}) is True
-79
View File
@@ -1,79 +0,0 @@
"""DoH resolver integration against the e2e platform's mock DoH responder.
The config-cluster analysis flagged DNS/DoH as a recurring break surface (#1028,
#108). A fully hermetic DoH-over-the-network profile isn't feasible in the HTTP
docker platform (DoH provider URLs are HTTPS + IP-pinned), so we exercise the
*real* ``DoHResolver`` client against the platform's mock ``doh`` role here, over
plain HTTP on localhost. This runs in normal CI (not just the nightly docker
matrix) and guards the DoH JSON-parsing path the app relies on.
"""
from __future__ import annotations
import importlib.util
import os
import threading
from pathlib import Path
from wsgiref.simple_server import WSGIServer, make_server
import pytest
MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py"
def _load_mock_doh_app(doh_map: str):
"""Import the platform mock_services module wired for the ``doh`` role.
The module wires its routes at import time from ``MOCK_ROLE``/``DOH_MAP``, so
those must be set before loading it.
"""
os.environ["MOCK_ROLE"] = "doh"
os.environ["DOH_MAP"] = doh_map
spec = importlib.util.spec_from_file_location("mock_doh_services", MOCK_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.app
@pytest.fixture(scope="module")
def doh_url():
if not MOCK_PATH.exists():
pytest.skip(f"platform mock not found at {MOCK_PATH}")
app = _load_mock_doh_app("aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11")
server: WSGIServer = make_server("127.0.0.1", 0, app)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
# Google JSON DoH style uses the /resolve endpoint.
yield f"http://127.0.0.1:{server.server_port}/resolve"
finally:
server.shutdown()
def _resolver(doh_url: str):
from shelfmark.download.network import DoHResolver
# hostname/ip args are the DoH server's own identity (used only for recursion
# avoidance); the localhost values here are irrelevant to the lookups under test.
return DoHResolver(doh_url, "127.0.0.1", "127.0.0.1")
def test_doh_resolves_mapped_host(doh_url) -> None:
"""The real DoH client parses the mock's JSON answer into an A record."""
assert _resolver(doh_url).resolve("aa.mock.test", "A") == ["172.30.0.10"]
def test_doh_nxdomain_returns_empty_not_error(doh_url) -> None:
"""An unmapped name yields an empty list (Status 3), not an exception —
the path that, when mishandled, surfaced as silent download failures."""
assert _resolver(doh_url).resolve("unmapped.invalid", "A") == []
def test_doh_resolver_caches_within_ttl(doh_url) -> None:
"""A second lookup is served from cache (the resolver's documented behaviour)."""
resolver = _resolver(doh_url)
first = resolver.resolve("cf.mock.test", "A")
assert first == ["172.30.0.11"]
assert ("cf.mock.test", "A") in resolver._cache
assert resolver.resolve("cf.mock.test", "A") == first
-132
View File
@@ -137,135 +137,3 @@ def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch):
assert result == [
(network.socket.AF_INET, network.socket.SOCK_STREAM, 6, "", ("203.0.113.10", 443))
]
def _addrinfo(ip):
return [(2, 1, 6, "", (ip, 443))]
class _FakeDoHResolver:
def __init__(self, ips):
self._ips = ips
def resolve(self, _hostname, _record_type):
return list(self._ips)
def test_build_detection_doh_resolver_uses_selected_provider(monkeypatch):
import shelfmark.download.network as network
monkeypatch.setattr(
network,
"DNS_PROVIDERS",
[
("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"),
],
)
monkeypatch.setattr(network, "_current_dns_index", 1) # user selected quad9
resolver = network._build_detection_doh_resolver()
assert resolver is not None
assert resolver.base_url == "https://dns.quad9.net/dns-query"
assert resolver.hostname == "dns.quad9.net"
assert resolver.ip == "9.9.9.9"
def test_build_detection_doh_resolver_falls_back_to_first_provider(monkeypatch):
import shelfmark.download.network as network
monkeypatch.setattr(
network,
"DNS_PROVIDERS",
[
("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
("quad9", ["9.9.9.9"], "https://dns.quad9.net/dns-query"),
],
)
monkeypatch.setattr(network, "_current_dns_index", -1) # system / not yet rotated
resolver = network._build_detection_doh_resolver()
assert resolver is not None
assert resolver.base_url == "https://cloudflare-dns.com/dns-query"
assert resolver.ip == "1.1.1.1"
def test_detect_dns_interference_flags_divergent_resolvers(monkeypatch):
import shelfmark.download.network as network
# System DNS (hijacked) returns an ISP block-page IP; DoH returns the real one.
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
monkeypatch.setattr(
network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
)
result = network.detect_dns_interference("annas-archive.pk")
assert result == {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
def test_detect_dns_interference_none_when_resolvers_agree(monkeypatch):
import shelfmark.download.network as network
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("203.0.113.7"))
monkeypatch.setattr(
network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
)
assert network.detect_dns_interference("annas-archive.pk") is None
def test_detect_dns_interference_none_when_doh_unavailable(monkeypatch):
import shelfmark.download.network as network
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
monkeypatch.setattr(network, "_build_detection_doh_resolver", lambda: None)
assert network.detect_dns_interference("annas-archive.pk") is None
def test_detect_dns_interference_skips_ip_and_local(monkeypatch):
import shelfmark.download.network as network
def _should_not_run():
raise AssertionError("resolver should not be built for IP/local hosts")
monkeypatch.setattr(network, "_build_detection_doh_resolver", _should_not_run)
assert network.detect_dns_interference("1.2.3.4") is None
assert network.detect_dns_interference("localhost") is None
def test_note_possible_dns_interference_warns_once_and_sets_flag(monkeypatch):
import shelfmark.download.network as network
monkeypatch.setattr(network, "_dns_interference_warned", set())
monkeypatch.setattr(network, "_dns_interference_active", False)
calls: list[str] = []
monkeypatch.setattr(
network,
"detect_dns_interference",
lambda host: (
calls.append(host) or {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
),
)
assert network.note_possible_dns_interference("annas-archive.pk") is True
assert network.dns_interference_detected() is True
# A repeat check for the same host must not re-run the costly detection.
assert network.note_possible_dns_interference("annas-archive.pk") is True
assert calls == ["annas-archive.pk"]
def test_note_possible_dns_interference_no_detection_keeps_flag_false(monkeypatch):
import shelfmark.download.network as network
monkeypatch.setattr(network, "_dns_interference_warned", set())
monkeypatch.setattr(network, "_dns_interference_active", False)
monkeypatch.setattr(network, "detect_dns_interference", lambda _host: None)
assert network.note_possible_dns_interference("annas-archive.pk") is False
assert network.dns_interference_detected() is False
@@ -1,87 +0,0 @@
"""Validate the e2e platform's webseed .torrent generator.
The ``full`` e2e profile relies on a tracker-less webseed torrent so a real
qBittorrent can complete a real download from the mock origin over HTTP. If the
generator emits malformed bencode or mismatched piece hashes, qBittorrent would
silently never complete — so we cross-check the generator against shelfmark's own
``bencode_decode`` / ``extract_info_hash_from_torrent`` here, in normal CI.
"""
from __future__ import annotations
import hashlib
import importlib.util
from pathlib import Path
import pytest
from shelfmark.download.clients.torrent_utils import (
bencode_decode,
extract_info_hash_from_torrent,
)
GEN_PATH = (
Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "make_webseed_torrent.py"
)
def _load_generator():
if not GEN_PATH.exists():
pytest.skip(f"generator not found at {GEN_PATH}")
spec = importlib.util.spec_from_file_location("make_webseed_torrent", GEN_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
PAYLOAD = b"E2E webseed payload \x00\x01\x02 " * 2000 # ~50 KiB -> multiple pieces
NAME = "sample-book.epub"
WEBSEED = "http://mock-aa/payload/sample-book.epub"
def test_generated_torrent_decodes_with_shelfmark_bencode() -> None:
gen = _load_generator()
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=16384)
decoded, _ = bencode_decode(raw)
assert isinstance(decoded, dict)
info = decoded[b"info"]
assert info[b"name"] == NAME.encode()
assert info[b"length"] == len(PAYLOAD)
# url-list (webseed) must point at the file the mock serves.
assert decoded[b"url-list"] == [WEBSEED.encode()]
# No tracker — the whole point is HTTP-only completion.
assert b"announce" not in decoded
def test_piece_hashes_match_payload_bytes() -> None:
gen = _load_generator()
piece_len = 16384
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=piece_len)
decoded, _ = bencode_decode(raw)
pieces = decoded[b"info"][b"pieces"]
expected = b"".join(
hashlib.sha1(PAYLOAD[i : i + piece_len]).digest() for i in range(0, len(PAYLOAD), piece_len)
)
assert pieces == expected, "piece hashes do not match payload — qbit would never complete"
assert len(pieces) % 20 == 0
def test_info_hash_matches_shelfmark_extractor() -> None:
"""Our infohash helper must agree with shelfmark's torrent parser."""
gen = _load_generator()
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
ours = gen.info_hash(raw)
theirs = extract_info_hash_from_torrent(raw)
assert theirs is not None
assert ours.lower() == theirs.lower(), (ours, theirs)
def test_generator_is_deterministic() -> None:
gen = _load_generator()
a = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
b = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
assert a == b, "torrent generation must be byte-deterministic for stable infohash"
-7
View File
@@ -1,7 +0,0 @@
# Runtime state created by run-e2e.sh: per-profile shelfmark logs, the app's
# /config (settings, users.db, secrets), downloaded books, and staging dirs.
.state/
# Python bytecode from the mock services + suite.
__pycache__/
*.pyc
-231
View File
@@ -1,231 +0,0 @@
# Shelfmark E2E Docker Testing Platform
A hermetic, container-based end-to-end platform that boots the real Shelfmark app
against **controllable** dependencies — a fake Anna's Archive, a Cloudflare gate, a
mock FlareSolverr bypasser, a mock Prowlarr + a real qBittorrent, custom DNS
servers, HTTP/SOCKS5 proxies, and a Tor profile — and runs a cluster test suite
under each **config profile**.
It exists to make the recurring bug clusters from the issue/PR analysis impossible
to reintroduce silently. The biggest one — Tor/Cloudflare/bypasser (37 issues /
67 fix PRs) — had almost no automated coverage; this platform changes that.
```
pytest suite (host :8084)
│
▼
shelfmark (under test) ── egress depends on the active profile:
├─ direct ──────────────► mock-aa (.10) fake Anna's Archive
├─ Cloudflare gate ─────► mock-cf (.11) ─► mock-aa [full: real Chrome solves it]
├─ FlareSolverr ────────► mock-cf (.11) ─► mock-flaresolverr (.12) [bypasser-external]
├─ custom DNS ──────────► coredns (.20) / coredns-blocked (.22) [dns-manual/blocked]
├─ HTTP / SOCKS proxy ──► tinyproxy (.30) / microsocks (.31) [proxy-http/socks]
├─ Tor (transparent) ───► in-image tor.sh [tor]
└─ Prowlarr → client ───► mock-prowlarr (.40) ─► qBittorrent [full: real download]
(all on one e2e docker network, 172.30.0.0/24, static IPs for DNS determinism)
```
## Quick start
```bash
# one profile
make e2e-platform # baseline
make e2e-platform-profile PROFILE=bypasser-external
make e2e-platform-profile PROFILE=client-deluge
make e2e-platform-full # heavy: real Chrome + DoH + real qBittorrent
# the whole matrix
make e2e-platform-matrix
# build the heavy image once, then reuse it (matrix does this automatically)
make e2e-platform-build
E2E_NO_BUILD=1 tests/e2e/platform/run-e2e.sh env/dns-doh.env
# debug: leave the stack up after the run
KEEP_UP=1 tests/e2e/platform/run-e2e.sh env/dns-blocked.env
```
Requirements: Docker + Compose v2, and `uv` (for the pytest runner). The runner
builds the Shelfmark image from the repo `Dockerfile`, boots the profile's stack,
waits for `/api/health`, runs the suite, and tears down. `run-matrix.sh` builds the
image **once** and reuses it across profiles (`E2E_NO_BUILD=1`) so the slow
xvfb/chromium layer isn't rebuilt per profile.
## How profiles work
Each profile is an env file in `env/`. It sets:
- `COMPOSE_PROFILES` — which optional services start (compose `profiles:`).
- `SM_*` — the app's config, injected as container env. Shelfmark treats
deployment ENV as authoritative (`config.get`: "Deployment-level ENV values
always win"), so a profile fully determines the app's DNS/proxy/bypasser/source
configuration with no runtime mutation.
- `E2E_PROFILE` — handed to pytest so the suite selects applicable tests.
Tests declare applicability with `@pytest.mark.profiles(...)`. **A test with no
marker is a profile-agnostic invariant and runs under every profile** — that is
how one cluster test ("source must be reachable") becomes the config matrix.
## The matrix (cluster × profile)
Status column: ✅ = run live on Docker and passing. Every profile below was run
end-to-end (`docker compose up` + suite + teardown) and passes.
| Profile | Egress / what it proves | Clusters | Regression targets | Status |
|---|---|---|---|---|
| `baseline` | Direct to fake AA; search/parse + #1028 clean-failure | 2,3,4 | #198 #293 #214 #1040 #1028 | ✅ 9 passed |
| `bypasser-external` | External bypasser wired; CF-gated search fails cleanly | 1 | #284 #202 #410 #369 | ✅ 5 passed |
| `bypasser-disabled` | CF-gated AA + bypasser OFF → no results (control) | 1 | #202 #410 | ✅ 4 passed |
| `dns-manual` | AA only resolvable via custom DNS (coredns) | config: DNS | #108 | ✅ 4 passed |
| `dns-blocked` | System DNS NXDOMAINs AA; custom DNS resolves it | config: DNS | **#1028** | ✅ 4 passed |
| `dns-doh` | System DNS blocks AA; **DoH over real HTTPS** resolves it | config: DoH | **#1028** #108 | ✅ 3 passed |
| `proxy-http` | All egress via tinyproxy, **proven by proxy logs** | config: proxy | **#956** | ✅ 6 passed |
| `proxy-socks` | All egress via SOCKS5 (microsocks), traversal-checked | config: proxy | #956 | ✅ 5 passed |
| `tor` | `USING_TOR=true` boots clean (restarts=0) | 1/6 Tor boot | #1021 #940 #801 | ✅ 5 passed |
| `client-transmission` | Prowlarr → **real Transmission** webseed download → /books | 5 clients | #1022 #634 | ✅ 4 passed |
| `client-deluge` | Prowlarr → **real Deluge** webseed download → /books | 5 clients | #530 | ✅ 4 passed |
| `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed |
| *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ |
> **The bypasser is download-time, not search-time.** Running the stack revealed
> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a
> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser
> (internal Chrome or external FlareSolverr) only runs during a file *download*
> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean*
> CF-gated-search failure, while the **`full` profile exercises the real end-to-end
> CF solve**: AA search/detail are reachable, but the AA slow-download link points
> at the gate, so downloading Moby-Dick forces the in-image headless Chromium to
> detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file —
> verified live (`Challenge detected: cloudflare` → `Bypass successful` → Moby-Dick
> in `/books`).
>
> **`bypasser-external` must set `SM_USING_EXTERNAL_BYPASSER=true`** — shelfmark does
> **not** derive it from `EXT_BYPASSER_URL`; without it the app silently uses the
> in-image Chrome bypasser instead of FlareSolverr.
Coverage of the 7 clusters from the analysis:
1. **Bypasser/Tor/Cloudflare** → `bypasser-external`, `bypasser-disabled`, `tor`.
2. **Search/metadata** → `baseline` (`test_cluster_search_aa.py`, hermetic via the
`direct_download` source so no external metadata provider is needed).
3. **AA parsing/mirrors** → `baseline` parse guards incl. the **layout-drift
fail-loud** test (#878/#879/#880).
4. **Permissions/file-move** → `baseline` (`test_cluster_download_permissions.py`).
5. **Torrent/usenet clients** → a mock Prowlarr + webseed torrent drives **three
real torrent clients** end to end (`full`=qBittorrent, `client-transmission`,
`client-deluge`) — completion detection + file move into `/books`. One
client-agnostic test (`test_cluster_clients.py`) covers all three.
6. **Docker/entrypoint/PUID-PGID** → profile-agnostic health + boot-log checks,
run under every profile.
7. **Audiobook/ABB** → parse-contract guards in
`tests/audiobookbay/test_scraper_contract.py` (info-hash normalization #386,
magnet fallback, layout drift). These run in **normal CI**, not the docker
matrix, because ABB hardcodes `https://` for its fetches (see Roadmap).
### Proxy traversal (not just reachability)
Because the app and the mock AA share the e2e network, a regression that ignores
the proxy config would still reach AA directly. `test_egress_actually_traverses_proxy`
drives a search and then inspects the proxy container's logs, so the proxy
profiles prove the egress *went through* the proxy — a real guard for #956.
### DoH — two layers
- **Offline** (`tests/download/test_doh_resolver_mock.py`, normal CI): the real
`DoHResolver` is driven against the mock `doh` role over localhost HTTP, covering
JSON-answer parsing, NXDOMAIN → empty, and caching.
- **In-stack** (`dns-doh` profile): the mock `doh` role serves the DNS JSON API over
**real HTTPS** (self-signed). The system resolver (coredns-blocked) NXDOMAINs
`aa.mock.test`, so the host can *only* be resolved via DoH; compose `extra_hosts`
redirects the `cloudflare-dns.com` provider to the in-stack mock and
`CERTIFICATE_VALIDATION=disabled` accepts the self-signed cert. The search reaching
AA proves the app's DoH path resolved the name end to end — **no app code change**.
### The `full` profile — real Chrome + real client (`make e2e-platform-full`)
The "everything real" heavy profile (test book: **Moby-Dick**), run nightly / on
demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, and
**passes live** (6 passed):
- **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are
reachable (`mock-aa`), but the AA *slow-download* link points at the Cloudflare
gate (`mock-cf`), whose challenge page runs JS that issues `cf_clearance` and
reloads. Downloading Moby-Dick forces the in-image headless Chromium (seleniumbase
CDP, in the `shelfmark` image via `xvfb`+`chromium`) to load the gate, detect the
challenge (`Challenge detected: cloudflare`), solve it (`_bypass_method_cdp_solve`),
and fetch the cleared "Download now" page → the file lands in `/books`. That
outcome is *only* reachable if Chrome solved the gate — the literal "spin a Chrome
browser" path and the strongest guard for the #1 cluster. Two subtleties this
surfaced, now handled by the mock: the cleared page must exceed the bypasser's
`_LOADING_BODY_LENGTH_MAX` (50 chars of innerText) or it loops as "still loading",
and the AA detail page must satisfy the brittle `original_nodes[-6]` parse (#880).
- **DoH** on at boot.
- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`,
`/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real
qBittorrent completes the download over HTTP (no tracker/peer/seeder), and
shelfmark's completion detection + file move lands the book in `/books`. The
webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked
against shelfmark's own bencode/infohash parser in
`tests/download/test_webseed_torrent_generator.py`), and the whole
prowlarr→qBittorrent path is configured declaratively via env (`env/full.env`).
## Components
| Path | Purpose |
|---|---|
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. |
| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. |
| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. |
| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. |
| `mocks/fixtures/*.html` | AA search/detail HTML in the **exact** shape the parser expects, plus drift/empty/no-files variants. |
| `docker-compose.e2e.yml` | The stack; optional services gated by compose profiles, static IPs for DNS determinism. |
| `dns/Corefile*`, `dns/mock.test.db` | coredns zones — working + ISP-block (NXDOMAIN). |
| `env/*.env` | The config profiles (matrix rows). |
| `suite/` | The pytest harness + cluster tests. |
| `run-e2e.sh` / `run-matrix.sh` | Boot one profile / loop the matrix. |
| `build-images.sh` | Build the heavy image once (`make e2e-platform-build`); reused via `E2E_NO_BUILD=1`. |
### Fault injection
The mock AA reproduces historical bugs deterministically. Injection rides inside
the search query as `E2EINJECT:<name>` (the app builds the AA URL itself and only
forwards the user query as `q=`). Names: `no_files`, `empty`, `layout_drift`,
`500`. The harness embeds them via `PlatformClient.direct_search(..., inject=...)`.
## Gating PRs (block merge on e2e failure)
The `.github/workflows/e2e-platform.yml` workflow runs on every PR. On a PR that
touches relevant code (`shelfmark/**`, `Dockerfile`, `entrypoint.sh`, `tor.sh`,
`tests/e2e/platform/**`) it runs the fast PR subset **and** the heavy `full`
profile (real Chrome solving Cloudflare + DoH + real qBittorrent), then a single
**`e2e required`** job aggregates them: it fails if any e2e job failed, and passes
(so it never hangs) when the e2e jobs are skipped on an unrelated PR.
The workflow producing a failing check is **not enough on its own** — GitHub only
*blocks merge* on checks listed in branch protection. A repo **admin** must, once:
- **UI:** Settings → Branches → branch protection rule for `main` →
*Require status checks to pass before merging* → add **`e2e required`**.
- **or `gh` (admin token):**
```bash
gh api -X PUT repos/calibrain/shelfmark/branches/main/protection \
-H "Accept: application/vnd.github+json" --input - <<'JSON'
{ "required_status_checks": { "strict": true, "contexts": ["e2e required"] },
"enforce_admins": true, "required_pull_request_reviews": null, "restrictions": null }
JSON
```
After that, any failure in the e2e platform tests (including the `full` profile)
blocks the PR from merging. Requiring just the one `e2e required` context covers
the whole dynamic matrix, so the list never needs updating as profiles change.
## Known limitations / follow-ups
- **rTorrent.** Not in the matrix: its rakshasa-libtorrent has **no GetRight/webseed
support**, so the hermetic webseed torrent (which qBittorrent/Transmission/Deluge
all complete) leaves rTorrent stuck at 0%. Supporting it needs a real tracker +
seeder (peer download) — a follow-up that the webseed design intentionally avoids.
- **Usenet clients (SABnzbd/NZBGet).** Not yet covered — completing a usenet download
hermetically needs a mock NNTP server serving the yEnc-encoded payload plus an NZB,
which is a separate (larger) build than the torrent webseed path.
- **Audiobook (cluster 7) in-stack.** ABB hardcodes `https://`, so it's covered
offline (`tests/audiobookbay/test_scraper_contract.py`); an in-stack
`audiobookbay` role needs the same self-signed-HTTPS plumbing the `dns-doh` profile
now uses for DoH.
-15
View File
@@ -1,15 +0,0 @@
#!/usr/bin/env bash
# Build every buildable image in the e2e stack once (the heavy `shelfmark` image
# plus the mock-* role images), so run-matrix.sh / run-e2e.sh with E2E_NO_BUILD=1
# can reuse them instead of rebuilding the xvfb/chromium layer per profile.
set -euo pipefail
PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$PLATFORM_DIR"
# Activate every profile that owns a buildable service so they all get built.
# (Download clients, coredns, proxies are pre-built images — nothing to build.)
export COMPOSE_PROFILES="bypasser-external,full,dns-doh"
echo "==> building shelfmark + mock images (one cold build of the chromium layer)"
docker compose -f docker-compose.e2e.yml build
echo "==> done. Reuse with: E2E_NO_BUILD=1 ./run-e2e.sh env/<profile>.env"
-13
View File
@@ -1,13 +0,0 @@
# coredns: authoritative for mock.test, forwards everything else.
mock.test:53 {
file /zones/mock.test.db
log
errors
}
. :53 {
forward . 1.1.1.1 8.8.8.8
cache 30
log
errors
}
-18
View File
@@ -1,18 +0,0 @@
# coredns: ISP-DNS-block simulation. Resolves nothing under mock.test
# (returns NXDOMAIN), so an app that relies on *system* DNS for book sources
# fails — while an app that uses its own custom DNS resolver still works.
# Regression harness for #1028 (internal bypasser used system DNS in subprocess).
mock.test:53 {
template IN ANY mock.test {
rcode NXDOMAIN
}
log
errors
}
. :53 {
# Everything else still resolves, so only the book-source host is "blocked".
forward . 1.1.1.1 8.8.8.8
log
errors
}
-13
View File
@@ -1,13 +0,0 @@
$ORIGIN mock.test.
$TTL 60
@ IN SOA ns.mock.test. admin.mock.test. (
1 ; serial
7200 ; refresh
3600 ; retry
1209600 ; expire
3600 ) ; minimum
@ IN NS ns.mock.test.
ns IN A 172.30.0.20
aa IN A 172.30.0.10
cf IN A 172.30.0.11
doh IN A 172.30.0.21
-332
View File
@@ -1,332 +0,0 @@
# Shelfmark e2e Docker testing platform
# ---------------------------------------------------------------------------
# A hermetic stack: the app under test plus *controllable* dependencies
# (fake Anna's Archive, a Cloudflare gate, a mock FlareSolverr bypasser, a DoH
# responder, a DNS server, HTTP + SOCKS5 proxies, and a Tor profile).
#
# Config profiles are env-driven: pick a profile with an env file under env/
# (it sets COMPOSE_PROFILES + the SM_* config the app boots with). The runner
# brings the stack up per profile, runs the matching cluster tests, tears down.
#
# ./run-e2e.sh env/baseline.env
# ./run-e2e.sh env/bypasser-external.env
# ./run-e2e.sh env/dns-blocked.env
#
# See README.md for the full cluster x profile matrix.
x-mock-build: &mock-build
build:
context: ./mocks
dockerfile: Dockerfile
networks:
e2e:
driver: bridge
ipam:
config:
- subnet: 172.30.0.0/24
services:
# ----- App under test --------------------------------------------------- #
shelfmark:
build:
context: ../../..
dockerfile: Dockerfile
target: ${SM_BUILD_TARGET:-shelfmark}
container_name: e2e-shelfmark
cap_add:
- NET_ADMIN # required by tor.sh iptables when USING_TOR=true
- NET_RAW
environment:
TZ: UTC
DEBUG: "true"
ONBOARDING: "false" # skip wizard; ephemeral storage
DISABLE_LOCAL_AUTH: "${SM_DISABLE_LOCAL_AUTH:-true}"
PUID: "${SM_PUID:-1000}"
PGID: "${SM_PGID:-1000}"
# --- source / mirror config (cluster 2/3) ---
DIRECT_DOWNLOAD_ENABLED: "${SM_DIRECT_DOWNLOAD_ENABLED:-true}"
AA_ADDITIONAL_URLS: "${SM_AA_URL:-http://mock-aa}"
# --- bypasser config (cluster 1) ---
USE_CF_BYPASS: "${SM_USE_CF_BYPASS:-false}"
# Selects the external (FlareSolverr) bypasser; when false the in-image
# Chrome (internal) bypasser is used. NOT derived from EXT_BYPASSER_URL.
USING_EXTERNAL_BYPASSER: "${SM_USING_EXTERNAL_BYPASSER:-false}"
EXT_BYPASSER_URL: "${SM_EXT_BYPASSER_URL:-}"
# --- prowlarr indexer + torrent client (cluster 5, `full` profile) ---
PROWLARR_ENABLED: "${SM_PROWLARR_ENABLED:-false}"
PROWLARR_URL: "${SM_PROWLARR_URL:-}"
PROWLARR_API_KEY: "${SM_PROWLARR_API_KEY:-}"
PROWLARR_TORRENT_CLIENT: "${SM_PROWLARR_TORRENT_CLIENT:-}"
QBITTORRENT_URL: "${SM_QBITTORRENT_URL:-}"
QBITTORRENT_USERNAME: "${SM_QBITTORRENT_USERNAME:-}"
QBITTORRENT_PASSWORD: "${SM_QBITTORRENT_PASSWORD:-}"
QBITTORRENT_CATEGORY: "${SM_QBITTORRENT_CATEGORY:-}"
DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT: "${SM_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT:-60}"
# transmission / deluge / rtorrent (client-* profiles)
TRANSMISSION_URL: "${SM_TRANSMISSION_URL:-}"
TRANSMISSION_USERNAME: "${SM_TRANSMISSION_USERNAME:-}"
TRANSMISSION_PASSWORD: "${SM_TRANSMISSION_PASSWORD:-}"
DELUGE_HOST: "${SM_DELUGE_HOST:-}"
DELUGE_PORT: "${SM_DELUGE_PORT:-}"
DELUGE_PASSWORD: "${SM_DELUGE_PASSWORD:-}"
RTORRENT_URL: "${SM_RTORRENT_URL:-}"
# --- DNS / DoH (config cluster) ---
CUSTOM_DNS: "${SM_CUSTOM_DNS:-}"
CUSTOM_DNS_MANUAL: "${SM_CUSTOM_DNS_MANUAL:-}"
USE_DOH: "${SM_USE_DOH:-false}"
# Disable TLS verification so the in-stack DoH-over-HTTPS mock (self-signed)
# is accepted in the dns-doh profile. Default keeps verification ON.
CERTIFICATE_VALIDATION: "${SM_CERTIFICATE_VALIDATION:-enabled}"
# --- proxy (config cluster) ---
PROXY_MODE: "${SM_PROXY_MODE:-none}"
HTTP_PROXY_URL: "${SM_HTTP_PROXY:-}"
HTTP_PROXY: "${SM_HTTP_PROXY:-}"
SOCKS5_PROXY: "${SM_SOCKS5_PROXY:-}"
NO_PROXY: "${SM_NO_PROXY:-}"
# --- tor (cluster 1/6) ---
USING_TOR: "${SM_USING_TOR:-false}"
ports:
- "8084:8084"
volumes:
- ./.state/config:/config
- ./.state/books:/books
- ${SM_DOWNLOADS_HOST_DIR:-./.state/downloads}:/downloads
- ./.state/tmp:/tmp/shelfmark
networks:
- e2e
dns:
# When a DNS profile is active, point the container's system resolver at
# our controllable server; otherwise Docker's embedded DNS (127.0.0.11).
- ${SM_SYSTEM_DNS:-127.0.0.11}
extra_hosts:
# Redirect the DoH provider hostname to the in-stack mock-doh (dns-doh
# profile). Harmless elsewhere — only the dns-doh profile enables DoH against
# the cloudflare provider, and /etc/hosts is consulted before the resolver.
- "cloudflare-dns.com:172.30.0.21"
restart: "no"
# ----- Fake Anna's Archive origin (always on) --------------------------- #
mock-aa:
<<: *mock-build
container_name: e2e-mock-aa
environment:
MOCK_ROLE: origin-aa
# When set (the `full` profile sets it to the CF gate), AA slow-download
# links point through Cloudflare so a real download forces the internal
# Chrome bypasser to solve the challenge. Empty -> same-origin (no CF).
SLOW_DOWNLOAD_BASE: "${SM_SLOW_DOWNLOAD_BASE:-}"
AA_FILE_BASE: "http://mock-aa"
networks:
e2e:
ipv4_address: 172.30.0.10
aliases:
- aa.mock.test
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://localhost/healthz')"]
interval: 3s
timeout: 3s
retries: 10
# ----- Cloudflare gate (profiles: bypasser-external, full) -------------- #
# `full` puts AA's slow-download behind this gate so the real Chrome bypasser
# must solve it; `bypasser-external` puts AA search behind it for the
# FlareSolverr negative/positive controls.
mock-cf:
<<: *mock-build
container_name: e2e-mock-cf
profiles: ["bypasser-external", "full"]
environment:
MOCK_ROLE: cloudflare
ORIGIN_INTERNAL_URL: http://mock-aa
networks:
e2e:
ipv4_address: 172.30.0.11
aliases:
- cf.mock.test
# ----- Mock FlareSolverr external bypasser (profile: bypasser-external) -- #
mock-flaresolverr:
<<: *mock-build
container_name: e2e-mock-flaresolverr
profiles: ["bypasser-external"]
environment:
MOCK_ROLE: flaresolverr
networks:
e2e:
ipv4_address: 172.30.0.12
# NOTE: the mock `doh` role lives in mock_services.py and is exercised by
# tests/download/test_doh_resolver_mock.py (real DoHResolver over localhost HTTP).
# An in-stack DoH service is intentionally absent — see README "Known limitations".
# ----- DNS server (profile: dns-manual) --------------------------------- #
coredns:
image: coredns/coredns:1.11.1
container_name: e2e-coredns
profiles: ["dns-manual"]
command: ["-conf", "/Corefile"]
volumes:
- ./dns/Corefile:/Corefile:ro
- ./dns/mock.test.db:/zones/mock.test.db:ro
networks:
e2e:
ipv4_address: 172.30.0.20
# ----- DNS server that NXDOMAINs the AA host (profile: dns-blocked) ------ #
# Simulates ISP DNS blocking (#1028); the app must fall back (DoH / direct).
coredns-blocked:
image: coredns/coredns:1.11.1
container_name: e2e-coredns-blocked
profiles: ["dns-blocked"]
command: ["-conf", "/Corefile"]
volumes:
- ./dns/Corefile.blocked:/Corefile:ro
networks:
e2e:
ipv4_address: 172.30.0.22
# ----- HTTP proxy (profile: proxy-http) --------------------------------- #
tinyproxy:
image: monokal/tinyproxy:latest
container_name: e2e-tinyproxy
profiles: ["proxy-http"]
command: ANY
networks:
e2e:
ipv4_address: 172.30.0.30
# ----- SOCKS5 proxy (profile: proxy-socks) ------------------------------ #
microsocks:
image: vimagick/microsocks:latest
container_name: e2e-microsocks
profiles: ["proxy-socks"]
networks:
e2e:
ipv4_address: 172.30.0.31
# ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- #
# Minimal Prowlarr API returning one torrent release whose .torrent is a
# webseed pointing at mock-aa. Drives the real torrent-client download in the
# `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles.
mock-prowlarr:
<<: *mock-build
container_name: e2e-mock-prowlarr
profiles: ["full", "client-transmission", "client-deluge", "client-qbittorrent-delayed"]
environment:
MOCK_ROLE: prowlarr
AA_INTERNAL_URL: http://mock-aa
networks:
e2e:
ipv4_address: 172.30.0.40
aliases:
- prowlarr.mock.test
# ----- Real qBittorrent download client (profiles: full, delayed path) --- #
# Auth is bypassed for the e2e subnet (qBittorrent.conf) so shelfmark connects
# without juggling the image's random temp password.
qbittorrent:
image: lscr.io/linuxserver/qbittorrent:latest
container_name: e2e-qbittorrent
profiles: ["full", "client-qbittorrent-delayed"]
environment:
PUID: "1000"
PGID: "1000"
TZ: UTC
WEBUI_PORT: "8080"
volumes:
- ./qbittorrent/qBittorrent.conf:/config/qBittorrent/qBittorrent.conf
# Normally shared with Shelfmark. In the delayed profile this points at a
# client-only directory and the sync sidecar makes it visible later.
- ${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/downloads}:/downloads
networks:
- e2e
# ----- Delayed file visibility sidecar (profile: client-qbittorrent-delayed) #
# Simulates a seedbox/remote-sync workflow: qBittorrent writes into one host
# directory while Shelfmark sees a separate /downloads directory that is synced
# only after a delay. This exercises DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT.
delayed-download-sync:
image: alpine:3.20
container_name: e2e-delayed-download-sync
profiles: ["client-qbittorrent-delayed"]
environment:
SYNC_DELAY_SECONDS: "${SM_DELAYED_DOWNLOAD_SYNC_SECONDS:-20}"
command:
- /bin/sh
- -c
- |
mkdir -p /synced
while true; do
found=0
for path in /remote/*; do
[ -e "$${path}" ] || continue
found=1
before="$$(du -sk /remote 2>/dev/null | awk '{print $$1}')"
sleep "$${SYNC_DELAY_SECONDS}"
after="$$(du -sk /remote 2>/dev/null | awk '{print $$1}')"
if [ -n "$${before}" ] && [ "$${before}" = "$${after}" ]; then
cp -a /remote/. /synced/ 2>/dev/null || true
fi
break
done
[ "$${found}" = "1" ] || sleep 1
done
volumes:
- ${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/downloads}:/remote:ro
- ${SM_DOWNLOADS_HOST_DIR:-./.state/downloads}:/synced
networks:
- e2e
# ----- Real Transmission client (profile: client-transmission) ---------- #
# Same webseed torrent + mock Prowlarr as `full`, different real client.
transmission:
image: lscr.io/linuxserver/transmission:latest
container_name: e2e-transmission
profiles: ["client-transmission"]
environment:
PUID: "1000"
PGID: "1000"
TZ: UTC
USER: admin
PASS: admin
volumes:
- ./.state/downloads:/downloads
networks:
- e2e
# ----- Real Deluge client (profile: client-deluge) ---------------------- #
# shelfmark talks to deluge-web (default WebUI password "deluge"), which
# connects to the bundled daemon — no daemon auth-file juggling.
deluge:
image: lscr.io/linuxserver/deluge:latest
container_name: e2e-deluge
profiles: ["client-deluge"]
environment:
PUID: "1000"
PGID: "1000"
TZ: UTC
volumes:
- ./.state/downloads:/downloads
networks:
- e2e
# NOTE: rTorrent is intentionally absent — its rakshasa-libtorrent has no
# GetRight/webseed support, so the hermetic webseed torrent can't drive it.
# Supporting it would need a real tracker + seeder (see README limitations).
# ----- DoH-over-HTTPS responder (profile: dns-doh) ---------------------- #
# In-stack DoH: serves the DNS JSON API over HTTPS with a self-signed cert.
# shelfmark reaches it because the profile (a) maps the DoH provider hostname
# to this container via extra_hosts and (b) sets CERTIFICATE_VALIDATION=disabled.
mock-doh:
<<: *mock-build
container_name: e2e-mock-doh
profiles: ["dns-doh"]
environment:
MOCK_ROLE: doh
DOH_TLS: "1"
DOH_MAP: "aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11"
networks:
e2e:
ipv4_address: 172.30.0.21
-8
View File
@@ -1,8 +0,0 @@
# Profile: baseline
# Direct connection to the fake Anna's Archive, no bypasser, no proxy, system DNS.
# Exercises clusters 2/3 (search + AA parsing) and the download happy path.
COMPOSE_PROFILES=
SM_AA_URL=http://mock-aa
SM_USE_CF_BYPASS=false
SM_PROXY_MODE=none
E2E_PROFILE=baseline
-9
View File
@@ -1,9 +0,0 @@
# Profile: bypasser-disabled (cluster 1 negative control)
# AA behind the Cloudflare gate but the bypasser is OFF. Proves the gate really
# challenges: search must FAIL clearly here, which is what makes the
# bypasser-external success meaningful (and guards #202/#410 "uses AA when CF
# bypass disabled" / "config not adhered").
COMPOSE_PROFILES=bypasser-external
SM_AA_URL=http://cf.mock.test
SM_USE_CF_BYPASS=false
E2E_PROFILE=bypasser-disabled
-13
View File
@@ -1,13 +0,0 @@
# Profile: bypasser-external (cluster 1)
# AA is served behind a Cloudflare gate. The app must detect the challenge and
# route through the external bypasser (mock FlareSolverr), which returns solved
# HTML + cf_clearance. Regression surface: #284 #226 #202 #1030 #410 #369 #267.
COMPOSE_PROFILES=bypasser-external
SM_AA_URL=http://cf.mock.test
SM_USE_CF_BYPASS=true
# Must be set explicitly — shelfmark does NOT derive it from EXT_BYPASSER_URL.
# Without it the app falls back to the in-image Chrome bypasser and never calls
# FlareSolverr, so this profile would silently exercise the wrong path.
SM_USING_EXTERNAL_BYPASSER=true
SM_EXT_BYPASSER_URL=http://mock-flaresolverr
E2E_PROFILE=bypasser-external
-13
View File
@@ -1,13 +0,0 @@
# Profile: client-deluge (cluster 5 — real Deluge client)
# shelfmark talks to deluge-web (default WebUI password "deluge"), which connects
# to the bundled daemon — no daemon auth-file juggling.
COMPOSE_PROFILES=client-deluge
SM_PROWLARR_ENABLED=true
SM_PROWLARR_URL=http://mock-prowlarr
SM_PROWLARR_API_KEY=e2e-test-key
SM_PROWLARR_TORRENT_CLIENT=deluge
SM_DELUGE_HOST=deluge
SM_DELUGE_PORT=8112
SM_DELUGE_PASSWORD=deluge
E2E_DOWNLOAD_TIMEOUT=180
E2E_PROFILE=client-deluge
-21
View File
@@ -1,21 +0,0 @@
# Profile: client-qbittorrent-delayed (cluster 5 — delayed completed-path visibility)
# Same mock Prowlarr + webseed torrent as `full`, but qBittorrent writes to a
# directory Shelfmark cannot see until the delayed sync sidecar copies it over.
# This models seedbox / remote-sync setups where the client reports completion
# before the completed path exists inside Shelfmark.
COMPOSE_PROFILES=client-qbittorrent-delayed
SM_PROWLARR_ENABLED=true
SM_PROWLARR_URL=http://mock-prowlarr
SM_PROWLARR_API_KEY=e2e-test-key
SM_PROWLARR_TORRENT_CLIENT=qbittorrent
SM_QBITTORRENT_URL=http://qbittorrent:8080
SM_QBITTORRENT_USERNAME=admin
SM_QBITTORRENT_PASSWORD=adminadmin
SM_QBITTORRENT_CATEGORY=books
# The sync delay must be longer than the default client path polling interval
# so Shelfmark actually enters the completed-path wait loop.
SM_DELAYED_DOWNLOAD_SYNC_SECONDS=20
SM_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT=90
E2E_DOWNLOAD_TIMEOUT=180
E2E_PROFILE=client-qbittorrent-delayed
-13
View File
@@ -1,13 +0,0 @@
# Profile: client-transmission (cluster 5 — real Transmission client)
# Same mock Prowlarr + webseed torrent as `full`, but the torrent goes to a real
# Transmission. Lean (no Chrome/CF/DoH), so it boots fast.
COMPOSE_PROFILES=client-transmission
SM_PROWLARR_ENABLED=true
SM_PROWLARR_URL=http://mock-prowlarr
SM_PROWLARR_API_KEY=e2e-test-key
SM_PROWLARR_TORRENT_CLIENT=transmission
SM_TRANSMISSION_URL=http://transmission:9091
SM_TRANSMISSION_USERNAME=admin
SM_TRANSMISSION_PASSWORD=admin
E2E_DOWNLOAD_TIMEOUT=180
E2E_PROFILE=client-transmission
-10
View File
@@ -1,10 +0,0 @@
# Profile: dns-blocked (config cluster: ISP DNS block — #1028 regression)
# The container's *system* resolver (coredns-blocked) NXDOMAINs the AA host,
# but the app's *custom* DNS resolver (coredns) resolves it fine. A correct app
# reaches AA via custom DNS; a regressed one (system DNS in subprocess) fails.
COMPOSE_PROFILES=dns-manual,dns-blocked
SM_AA_URL=http://aa.mock.test
SM_SYSTEM_DNS=172.30.0.22
SM_CUSTOM_DNS=manual
SM_CUSTOM_DNS_MANUAL=172.30.0.20
E2E_PROFILE=dns-blocked

Some files were not shown because too many files have changed in this diff Show More