Compare commits

..
Author SHA1 Message Date
Alex 7a4c913809 Clarify Docker digest update group 2026-05-12 12:14:29 +01:00
Alex 230015ad57 Add Dependabot update cooldown 2026-05-12 12:09:27 +01:00
Alex 28bf26414d Run uv Dependabot checks daily 2026-05-12 11:30:29 +01:00
209 changed files with 1941 additions and 17542 deletions
-24
View File
@@ -42,13 +42,7 @@ updates:
open-pull-requests-limit: 5
groups:
docker-base-image-digests:
# Exclude python from the group on purpose. Dependabot's Docker
# pre-release filter is bypassed for *grouped* updates
# (dependabot-core#9496), so a grouped python update proposes pre-release
# tags like python:3.15.0b2 as if they were a normal stable minor bump.
# node + uv stay grouped into a single digest PR.
patterns: ["*"]
exclude-patterns: ["python"]
ignore:
# Node.js: block major-version bumps so dependabot never proposes
# moving from one LTS line to a non-LTS "Current" release (e.g. 24 -> 25).
@@ -56,24 +50,6 @@ updates:
- dependency-name: "node"
update-types: ["version-update:semver-major"]
# Python: block minor/major bumps. Ungrouping python (above) is NOT enough
# to keep pre-releases out — dependabot-core#13815 rewrote the Docker
# pre-release heuristic to catch PEP 440 tags like 3.15.0a2 / 3.5.0b3, but
# the suffixed real tag still slipped through as PR #1169
# (python:3.14.6-slim -> python:3.15.0b3-slim). CPython spells
# pre-releases without a separator, so tag parsing reads 3.15.0b3 as an
# ordinary version that sorts above 3.14.6.
#
# A minor-version ignore blocks it regardless of spelling. Patch bumps
# (3.14.6 -> 3.14.7) and same-tag digest refreshes still land automatically.
# Moving the runtime to a new Python minor is a manual, deliberate change:
# bump the tag here and confirm C-extension wheels (greenlet/gevent) exist
# for it — a source build against a pre-release ABI boots an app that binds
# its port but never serves, which wedges e2e for the full 6h job limit.
- dependency-name: "python"
update-types:
["version-update:semver-major", "version-update:semver-minor"]
# GitHub Actions
- package-ecosystem: "github-actions"
directory: "/"
@@ -67,10 +67,10 @@ jobs:
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Log in to the Container registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -78,13 +78,7 @@ jobs:
- name: Extract metadata for ${{ matrix.target }} image
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
env:
# Annotate both the per-platform manifests and the multi-arch image
# index. The index level is what manifest-list consumers (Renovate's
# minimumReleaseAge soak check, provenance/SBOM tooling) read for the
# standard org.opencontainers.image.* annotations, including `created`.
DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
tags: |
@@ -96,11 +90,11 @@ jobs:
type=ref,event=tag
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build and push ${{ matrix.target }} Docker image
id: push
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
platforms: linux/amd64,linux/arm64
context: .
@@ -111,11 +105,10 @@ jobs:
RELEASE_VERSION=${{ github.ref_name }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
annotations: ${{ steps.meta.outputs.annotations }}
- name: Generate artifact attestation for ${{ matrix.target }} image
if: github.event_name != 'pull_request'
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
subject-digest: ${{ steps.push.outputs.digest }}
@@ -134,14 +127,14 @@ jobs:
LEGACY_NAME: calibre-web-automated-book-downloader
steps:
- name: Log in to registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Create legacy aliases
run: |
+15 -15
View File
@@ -13,10 +13,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install uv and Python
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.3"
python-version: "3.14"
@@ -39,10 +39,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install uv and Python
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.3"
python-version: "3.14"
@@ -59,10 +59,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install uv and Python
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.3"
python-version: "3.14"
@@ -78,13 +78,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build shelfmark-lite image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
target: shelfmark-lite
@@ -99,10 +99,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: "npm"
@@ -122,10 +122,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: "npm"
@@ -142,10 +142,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: "npm"
+4 -4
View File
@@ -22,17 +22,17 @@ jobs:
language: [python, javascript-typescript]
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Initialize CodeQL
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
uses: github/codeql-action/autobuild@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
with:
category: "/language:${{ matrix.language }}"
-135
View File
@@ -1,135 +0,0 @@
name: E2E Platform
# Hermetic end-to-end matrix: boots the app under test against mock
# Anna's Archive / Cloudflare / bypasser / DNS / proxy / Tor / real torrent
# clients and runs the cluster suite under each config profile.
#
# On a PR that touches relevant code, this runs a fast core subset *and* the heavy
# `full` profile (real Chrome solving Cloudflare + DoH + real qBittorrent). The
# `e2e-required` job aggregates them into ONE status check — make that check a
# required status check in branch protection to block merges on any e2e failure
# (see tests/e2e/platform/README.md "Gating PRs").
on:
pull_request:
schedule:
- cron: "0 4 * * *" # nightly full matrix
workflow_dispatch:
concurrency:
group: e2e-platform-${{ github.ref }}
cancel-in-progress: true
jobs:
# Detect whether anything that affects the e2e platform changed. This lets the
# required check always report (never stuck "pending") while only spending CI on
# PRs that can actually break the e2e stack.
changes:
runs-on: ubuntu-latest
outputs:
relevant: ${{ steps.filter.outputs.relevant }}
steps:
- uses: actions/checkout@v7
- uses: dorny/paths-filter@v4.0.3
id: filter
with:
filters: |
relevant:
- 'shelfmark/**'
- 'entrypoint.sh'
- 'tor.sh'
- 'Dockerfile'
- 'tests/e2e/platform/**'
- '.github/workflows/e2e-platform.yml'
select-profiles:
needs: changes
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
runs-on: ubuntu-latest
outputs:
profiles: ${{ steps.pick.outputs.profiles }}
steps:
- id: pick
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
echo 'profiles=["baseline","bypasser-external","dns-blocked"]' >> "$GITHUB_OUTPUT"
else
echo 'profiles=["baseline","bypasser-external","bypasser-disabled","dns-manual","dns-blocked","dns-doh","proxy-http","proxy-socks","tor","client-transmission","client-deluge","client-qbittorrent-delayed"]' >> "$GITHUB_OUTPUT"
fi
e2e:
needs: select-profiles
runs-on: ubuntu-latest
# A wedged app under test must not burn GitHub's 6h max job limit. A healthy
# profile run finishes in ~3-5 min; anything past 25 is hung, not slow.
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }}
name: e2e (${{ matrix.profile }})
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install uv and Python
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
python-version: "3.14"
enable-cache: true
- name: Sync dependencies
run: make install-python-dev
- name: Run e2e platform (${{ matrix.profile }})
run: tests/e2e/platform/run-e2e.sh env/${{ matrix.profile }}.env
- name: Dump shelfmark logs on failure
if: failure()
run: cat tests/e2e/platform/.state/shelfmark.${{ matrix.profile }}.log || true
# Heavy "everything real" job: real Chrome internal bypasser solving Cloudflare +
# DoH + real qBittorrent webseed download. Runs on relevant PRs and nightly.
e2e-full:
needs: changes
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
runs-on: ubuntu-latest
# Real Chrome + qBittorrent is the slowest profile; still nowhere near 40 min.
timeout-minutes: 40
name: e2e (full — real Chrome + qBittorrent)
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install uv and Python
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
python-version: "3.14"
enable-cache: true
- name: Sync dependencies
run: make install-python-dev
- name: Run full pipeline
run: tests/e2e/platform/run-e2e.sh env/full.env
- name: Dump logs on failure
if: failure()
run: |
cat tests/e2e/platform/.state/shelfmark.full.log || true
docker logs e2e-qbittorrent || true
# Single aggregated gate. ALWAYS runs (so a required check never hangs "pending"
# on unrelated PRs) and FAILS if any e2e job failed/was cancelled. Make THIS the
# required status check in branch protection.
e2e-required:
name: e2e required
needs: [e2e, e2e-full]
if: always()
runs-on: ubuntu-latest
steps:
- name: Aggregate e2e results
run: |
matrix='${{ needs.e2e.result }}'
full='${{ needs.e2e-full.result }}'
echo "e2e matrix=$matrix, e2e-full=$full"
# success or skipped (unrelated PR) is OK; failure/cancelled blocks.
for r in "$matrix" "$full"; do
if [ "$r" = "failure" ] || [ "$r" = "cancelled" ]; then
echo "::error::An e2e platform job did not pass — blocking."
exit 1
fi
done
echo "All e2e platform jobs passed (or were skipped as not relevant)."
-4
View File
@@ -166,10 +166,6 @@ ENV/
env.bak/
venv.bak/
# ...but the e2e platform test profiles live in an env/ dir and must be tracked
!tests/e2e/platform/env/
!tests/e2e/platform/env/*.env
# Spyder project settings
.spyderproject
.spyproject
+11 -54
View File
@@ -4,7 +4,7 @@ ARG BUILDPLATFORM
ARG BUILDARCH
# Frontend build stage.
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 AS frontend-builder
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f AS frontend-builder
# Helpful debug output to see what platforms BuildKit thinks it's using
RUN echo "BUILDPLATFORM=$BUILDPLATFORM BUILDARCH=$BUILDARCH TARGETPLATFORM=$TARGETPLATFORM TARGETARCH=$TARGETARCH"
@@ -25,7 +25,7 @@ COPY src/frontend/ ./
RUN npm run build
# Use python-slim as the base image
FROM python:3.14.7-slim@sha256:83c1cebb322d099ac9e3a3a532ba74b0146d702838b25e4c75c02fa81ffeb910 AS base
FROM python:3.14-slim@sha256:1697e8e8d39bf168e177ac6b5fdab6df86d81cfc24dae17dfb96cfc3ef76b4dd AS base
COPY --from=ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 /uv /uvx /bin/
@@ -59,11 +59,6 @@ ENV FLASK_PORT=8084
# Configure locale, timezone, and perform initial cleanup in a single layer
RUN apt-get update && \
apt-get install -y --no-install-recommends \
# For building C-extensions (cffi, gevent, etc.)
gcc \
g++ \
libffi-dev \
python3-dev \
# For locale
locales tzdata \
# For healthcheck
@@ -77,12 +72,7 @@ RUN apt-get update && \
# --- Tor support (activated via USING_TOR=true) ---
tor \
supervisor \
iptables \
# --- WireGuard support (activated via USING_WIREGUARD=true) ---
wireguard-tools \
iproute2 \
procps \
ca-certificates && \
iptables && \
# Configure iptables alternatives for tor.sh compatibility
update-alternatives --set iptables /usr/sbin/iptables-legacy && \
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
@@ -141,7 +131,7 @@ RUN mkdir -p \
ln -s /tmp/shelfmark/seleniumbase/archived_files /app/archived_files && \
chown -R 1000:1000 /config /books /home/shelfmark /tmp/shelfmark /var/log/shelfmark && \
chmod -R a+rX /app && \
chmod +x /app/entrypoint.sh /app/tor.sh /app/wireguard.sh /app/genDebug.sh
chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
# Expose the application port
EXPOSE ${FLASK_PORT}
@@ -157,39 +147,21 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
FROM base AS shelfmark
# --- Chromium (PINNED to 149.0.7827.196) ---
# Debian's chromium 150.0.7871.46-1~deb13u1 security update (trixie-security,
# 2026-07-05) no longer opens the DevTools remote-debugging TCP port at all
# (no listener, no DevToolsActivePort file, even with a custom --user-data-dir;
# the RemoteDebuggingAllowed policy does not restore it). The SeleniumBase
# Pure-CDP driver connects through that port (/json/version), so with 150 every
# internal bypass dies with "Pure CDP browser startup failed" and all
# CF-gated downloads fail. Install the last working version from
# snapshot.debian.org until the bypasser can talk to Chromium >= 150 (e.g.
# pipe-based DevTools / UC mode) or seleniumbase ships a fix.
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
# This flag is set in internal_bypasser.py _get_browser_args()
ARG CHROMIUM_VERSION=149.0.7827.196-1~deb13u1
ARG CHROMIUM_SNAPSHOT=20260704T000000Z
RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/${CHROMIUM_SNAPSHOT}/ trixie-security main" \
> /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
apt-get update -o Acquire::Retries=5 && \
apt-get install -y --no-install-recommends -o Acquire::Retries=5 \
RUN apt-get update && \
apt-get install -y --no-install-recommends \
# For dumb display
xvfb \
# For screen recording
ffmpeg \
chromium=${CHROMIUM_VERSION} \
chromium-common=${CHROMIUM_VERSION} \
# --- Chromium (unpinned - uses latest from Debian repos) ---
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
# This flag is set in internal_bypasser.py _get_browser_args()
chromium \
chromium-common \
# For tkinter (pyautogui)
python3-tk \
# For RAR extraction
unrar-free && \
# Keep apt from "upgrading" chromium past the pin inside derived images
printf 'Package: chromium chromium-common\nPin: version %s\nPin-Priority: 1001\n' "${CHROMIUM_VERSION}" \
> /etc/apt/preferences.d/chromium-pin && \
rm /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
# Create symlink so rarfile library can find unrar
ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
# Cleanup APT cache
@@ -201,21 +173,6 @@ RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --locked --no-default-groups --extra browser
# Deterministically resolve the Xlib namespace collision.
# pyautogui/mouseinfo pull the stale `python3-xlib` (0.15, 2014), while the
# `--extra browser` set pulls `python-xlib` (0.33). Both packages install into
# the same top-level `Xlib/` namespace, so whichever lands last wins. When the
# 2014 build wins, `Xlib.X` is missing `FamilyServerInterpreted`, which the
# SeleniumBase Pure-CDP driver requires at browser startup -> every bypass fails
# with "module 'Xlib.X' has no attribute 'FamilyServerInterpreted'" and no
# Cloudflare/DDoS-Guard protected download can complete. Drop the stale package
# and force python-xlib 0.33 to own the namespace. pyautogui runs fine against
# 0.33 (superset API).
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip uninstall --python /app/.venv/bin/python python3-xlib && \
uv pip install --python /app/.venv/bin/python --reinstall python-xlib==0.33 && \
/app/.venv/bin/python -c "import Xlib.X; assert hasattr(Xlib.X, 'FamilyServerInterpreted'), 'Xlib.X.FamilyServerInterpreted missing after fix'; print('Xlib namespace OK:', Xlib.__version__)"
# uv is only needed while building the image.
RUN rm -f /usr/bin/uv /usr/bin/uvx
+1 -29
View File
@@ -1,4 +1,4 @@
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov e2e-platform e2e-platform-profile e2e-platform-matrix e2e-platform-full e2e-platform-build checks fix
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov checks fix
# Frontend directory
FRONTEND_DIR := src/frontend
@@ -38,10 +38,6 @@ help:
@echo " python-checks - Run all Python static analysis checks"
@echo " python-test - Run unit tests"
@echo " python-test-cov - Run unit tests with coverage report"
@echo " e2e-platform - Run e2e docker platform (baseline profile)"
@echo " e2e-platform-profile PROFILE=<name> - Run e2e platform for one profile"
@echo " e2e-platform-matrix - Run e2e platform across all config profiles"
@echo " e2e-platform-full - Run heavy 'full' profile (real Chrome bypasser + DoH + real qBittorrent)"
@echo " clean - Remove node_modules and build artifacts"
@echo ""
@echo "Backend (Docker):"
@@ -131,30 +127,6 @@ python-test-cov:
@echo "Running tests with coverage..."
uv run pytest tests/ -x --tb=short -m "not integration and not e2e" --cov --cov-report=term-missing
# E2E docker platform: hermetic stack (mock AA/Cloudflare/bypasser/DNS/proxy/Tor)
# exercised across config profiles. See tests/e2e/platform/README.md.
E2E_PLATFORM_DIR := tests/e2e/platform
e2e-platform:
@echo "Running e2e platform (baseline profile)..."
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/baseline.env
e2e-platform-profile:
@echo "Running e2e platform (profile=$(PROFILE))..."
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/$(PROFILE).env
e2e-platform-matrix:
@echo "Running e2e platform matrix (all profiles)..."
cd $(E2E_PLATFORM_DIR) && ./run-matrix.sh
e2e-platform-build:
@echo "Pre-building e2e platform images once (reused across profiles)..."
cd $(E2E_PLATFORM_DIR) && ./build-images.sh
e2e-platform-full:
@echo "Running e2e platform FULL profile (real Chrome bypasser + DoH + real qBittorrent)..."
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/full.env
# Frontend linting
frontend-lint:
@echo "Running Oxlint..."
-43
View File
@@ -1,43 +0,0 @@
# Routes all traffic through a WireGuard tunnel - requires root startup.
#
# Mount your wg-quick config at /config/wg0.conf (read-only is fine). All
# non-LAN egress is forced through the tunnel by an iptables kill-switch, so if
# the tunnel drops, external traffic fails closed. LAN ranges (WebUI + internal
# download clients like Prowlarr / qBittorrent) stay reachable off-tunnel.
services:
shelfmark-wireguard:
image: ghcr.io/calibrain/shelfmark:latest
environment:
FLASK_PORT: 8084
# Quoted so it is passed as the literal string "true": entrypoint.sh compares
# $USING_WIREGUARD against "true", and some Compose implementations stringify
# a bare YAML boolean as "True", which would silently NOT enable WireGuard.
USING_WIREGUARD: "true"
# Path to the mounted wg-quick config (default shown).
WIREGUARD_CONFIG: /config/wg0.conf
# CIDRs kept OFF the tunnel so the WebUI and internal clients stay reachable.
LAN_NETWORK: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
PUID: 1000
PGID: 1000
cap_add:
- NET_ADMIN
- NET_RAW
# WireGuard needs the module/kernel routing; NET_ADMIN covers wg-quick.
sysctls:
- net.ipv4.conf.all.src_valid_mark=1
# Disable IPv6 in the container so the kill-switch can guarantee no IPv6
# leak path on kernels/containers without a usable ip6tables. wireguard.sh
# fails closed if IPv6 is neither kill-switched nor disabled. If your host
# DOES have a working ip6tables you may omit these (an ip6tables kill-switch
# is installed instead); or set WIREGUARD_ALLOW_IPV6_LEAK=true only if the
# container genuinely has no IPv6 connectivity.
- net.ipv6.conf.all.disable_ipv6=1
- net.ipv6.conf.default.disable_ipv6=1
ports:
- 8084:8084
restart: unless-stopped
volumes:
- /path/to/books:/books # Default destination for book downloads
- /path/to/config:/config # App configuration (put wg0.conf here)
# Required for torrent / usenet - path must match your download client's volume exactly
# - /path/to/downloads:/path/to/downloads
+53 -60
View File
@@ -1,79 +1,72 @@
[
{ "language": "English", "code": "en", "aliases": ["eng"] },
{ "language": "Chinese", "code": "zh", "aliases": ["chi", "zho"] },
{ "language": "Russian", "code": "ru", "aliases": ["rus"] },
{ "language": "Spanish", "code": "es", "aliases": ["spa"] },
{ "language": "French", "code": "fr", "aliases": ["fra", "fre"] },
{ "language": "German", "code": "de", "aliases": ["deu", "ger"] },
{ "language": "Italian", "code": "it", "aliases": ["ita"] },
{ "language": "Portuguese", "code": "pt", "aliases": ["por"] },
{ "language": "Polish", "code": "pl", "aliases": ["pol"] },
{ "language": "Bulgarian", "code": "bg", "aliases": ["bul"] },
{ "language": "Dutch", "code": "nl", "aliases": ["dut", "nld"] },
{ "language": "Japanese", "code": "ja", "aliases": ["jap", "jpn"] },
{ "language": "Arabic", "code": "ar", "aliases": ["ara"] },
{ "language": "Hebrew", "code": "he", "aliases": ["heb"] },
{ "language": "Turkish", "code": "tr", "aliases": ["tur"] },
{ "language": "Hungarian", "code": "hu", "aliases": ["hun"] },
{ "language": "Latin", "code": "la", "aliases": ["lat"] },
{ "language": "Czech", "code": "cs", "aliases": ["ces", "cze"] },
{ "language": "Korean", "code": "ko", "aliases": ["kor"] },
{ "language": "Ukrainian", "code": "uk", "aliases": ["ukr"] },
{ "language": "Indonesian", "code": "id", "aliases": ["ind"] },
{ "language": "Romanian", "code": "ro", "aliases": ["rom", "ron"] },
{ "language": "Swedish", "code": "sv", "aliases": ["swe"] },
{ "language": "Greek", "code": "el", "aliases": ["ell", "gre"] },
{ "language": "Lithuanian", "code": "lt", "aliases": ["lit"] },
{ "language": "Bangla", "code": "bn", "aliases": ["ben", "bengali"] },
{ "language": "Traditional Chinese", "code": "zh-Hant", "aliases": ["zh‑Hant"] },
{ "language": "Afrikaans", "code": "af", "aliases": ["afr"] },
{ "language": "Catalan", "code": "ca", "aliases": ["cat"] },
{ "language": "Danish", "code": "da", "aliases": ["dan"] },
{ "language": "Thai", "code": "th", "aliases": ["tha"] },
{ "language": "Hindi", "code": "hi", "aliases": ["hin"] },
{ "language": "Irish", "code": "ga", "aliases": ["gle"] },
{ "language": "Latvian", "code": "lv", "aliases": ["lav"] },
{ "language": "English", "code": "en" },
{ "language": "Chinese", "code": "zh" },
{ "language": "Russian", "code": "ru" },
{ "language": "Spanish", "code": "es" },
{ "language": "French", "code": "fr" },
{ "language": "German", "code": "de" },
{ "language": "Italian", "code": "it" },
{ "language": "Portuguese", "code": "pt" },
{ "language": "Polish", "code": "pl" },
{ "language": "Bulgarian", "code": "bg" },
{ "language": "Dutch", "code": "nl" },
{ "language": "Japanese", "code": "ja" },
{ "language": "Arabic", "code": "ar" },
{ "language": "Hebrew", "code": "he" },
{ "language": "Turkish", "code": "tr" },
{ "language": "Hungarian", "code": "hu" },
{ "language": "Latin", "code": "la" },
{ "language": "Czech", "code": "cs" },
{ "language": "Korean", "code": "ko" },
{ "language": "Ukrainian", "code": "uk" },
{ "language": "Indonesian", "code": "id" },
{ "language": "Romanian", "code": "ro" },
{ "language": "Swedish", "code": "sv" },
{ "language": "Greek", "code": "el" },
{ "language": "Lithuanian", "code": "lt" },
{ "language": "Bangla", "code": "bn" },
{ "language": "Traditional Chinese", "code": "zh‑Hant" },
{ "language": "Afrikaans", "code": "af" },
{ "language": "Catalan", "code": "ca" },
{ "language": "Danish", "code": "da" },
{ "language": "Thai", "code": "th" },
{ "language": "Hindi", "code": "hi" },
{ "language": "Irish", "code": "ga" },
{ "language": "Latvian", "code": "lv" },
{ "language": "Tibetan", "code": "bo" },
{ "language": "Kannada", "code": "kn", "aliases": ["kan"] },
{ "language": "Serbian", "code": "sr", "aliases": ["srp"] },
{ "language": "Persian", "code": "fa", "aliases": ["farsi", "fas", "per"] },
{ "language": "Croatian", "code": "hr", "aliases": ["hrv"] },
{ "language": "Kannada", "code": "kn" },
{ "language": "Serbian", "code": "sr" },
{ "language": "Persian", "code": "fa" },
{ "language": "Croatian", "code": "hr" },
{ "language": "Slovak", "code": "sk" },
{ "language": "Javanese", "code": "jv", "aliases": ["jav"] },
{ "language": "Vietnamese", "code": "vi", "aliases": ["vie"] },
{ "language": "Urdu", "code": "ur", "aliases": ["urd"] },
{ "language": "Finnish", "code": "fi", "aliases": ["fin"] },
{ "language": "Norwegian", "code": "no", "aliases": ["nor"] },
{ "language": "Javanese", "code": "jv" },
{ "language": "Vietnamese", "code": "vi" },
{ "language": "Urdu", "code": "ur" },
{ "language": "Finnish", "code": "fi" },
{ "language": "Norwegian", "code": "no" },
{ "language": "Kinyarwanda", "code": "rw" },
{ "language": "Tamil", "code": "ta", "aliases": ["tam"] },
{ "language": "Tamil", "code": "ta" },
{ "language": "Belarusian", "code": "be" },
{ "language": "Kazakh", "code": "kk" },
{ "language": "Mongolian", "code": "mn" },
{ "language": "Georgian", "code": "ka" },
{ "language": "Slovenian", "code": "sl", "aliases": ["slv"] },
{ "language": "Slovenian", "code": "sl" },
{ "language": "Esperanto", "code": "eo" },
{ "language": "Galician", "code": "gl" },
{ "language": "Marathi", "code": "mr", "aliases": ["mar"] },
{ "language": "Filipino", "code": "fil", "aliases": ["tagalog", "tgl"] },
{ "language": "Gujarati", "code": "gu", "aliases": ["guj"] },
{ "language": "Malayalam", "code": "ml", "aliases": ["mal"] },
{ "language": "Marathi", "code": "mr" },
{ "language": "Filipino", "code": "fil" },
{ "language": "Gujarati", "code": "gu" },
{ "language": "Malayalam", "code": "ml" },
{ "language": "Kyrgyz", "code": "ky" },
{ "language": "Azerbaijani", "code": "az" },
{ "language": "Quechua", "code": "qu" },
{ "language": "Swahili", "code": "sw" },
{ "language": "Bashkir", "code": "ba" },
{ "language": "Punjabi", "code": "pa", "aliases": ["pan"] },
{ "language": "Malay", "code": "ms", "aliases": ["may", "msa"] },
{ "language": "Telugu", "code": "te", "aliases": ["tel"] },
{ "language": "Punjabi", "code": "pa" },
{ "language": "Malay", "code": "ms" },
{ "language": "Telugu", "code": "te" },
{ "language": "Albanian", "code": "sq" },
{ "language": "Uyghur", "code": "ug" },
{ "language": "Armenian", "code": "hy" },
{ "language": "Shan", "code": "shn" },
{ "language": "Bosnian", "code": "bs", "aliases": ["bos"] },
{ "language": "Burmese", "code": "my", "aliases": ["bur", "mya"] },
{ "language": "Estonian", "code": "et", "aliases": ["est"] },
{ "language": "Icelandic", "code": "is", "aliases": ["ice", "isl"] },
{ "language": "Manx", "code": "gv", "aliases": ["glv"] },
{ "language": "Scottish Gaelic", "code": "gd", "aliases": ["gla"] },
{ "language": "Sanskrit", "code": "sa", "aliases": ["san"] }
{ "language": "Shan", "code": "shn" }
]
-25
View File
@@ -1,25 +0,0 @@
# Local development - WireGuard variant
services:
shelfmark-wireguard-dev:
extends:
file: ./compose/docker-compose.wireguard.yml
service: shelfmark-wireguard
build:
context: .
dockerfile: Dockerfile
target: shelfmark
environment:
# Quoted so they are passed as the literal string "true" (entrypoint.sh and
# the app compare against "true"); a bare YAML boolean can be stringified as
# "True" by some Compose variants, silently disabling the feature.
DEBUG: "true"
USING_WIREGUARD: "true"
WIREGUARD_CONFIG: /config/wg0.conf
volumes:
- ./.local/config:/config
- ./.local/books:/books
- ./.local/log:/var/log/shelfmark
- ./.local/tmp:/tmp/shelfmark
# Place your wg-quick config at ./.local/config/wg0.conf
# Required for torrent / usenet - path must match your download client's volume exactly
# - /path/to/downloads:/path/to/downloads
-4
View File
@@ -91,10 +91,6 @@ Example:
- Shelfmark can see the same files at `/downloads/books/...`
- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
If the files are copied or synced into Shelfmark on a delay, increase **Completed Path Wait (seconds)**
in Settings -> Advanced. The default is 60 seconds; seedbox or remote-sync setups may need a value
longer than the sync interval.
## File Processing Options
### Transfer Method (Torrent / Usenet Only)
+18 -256
View File
@@ -7,7 +7,6 @@ This document lists all configuration options that can be set via environment va
## Table of Contents
- [Bootstrap Configuration](#bootstrap-configuration)
- [Egress / VPN Routing](#egress--vpn-routing)
- [General](#general)
- [Search Mode](#search-mode)
- [Downloads](#downloads)
@@ -23,7 +22,6 @@ This document lists all configuration options that can be set via environment va
- [Hardcover](#metadata-providers-hardcover)
- [Open Library](#metadata-providers-open-library)
- [Google Books](#metadata-providers-google-books)
- [Moly.hu](#metadata-providers-moly.hu)
- [Direct Download](#direct-download)
- [Download Sources](#direct-download-download-sources)
- [Cloudflare Bypass](#direct-download-cloudflare-bypass)
@@ -147,98 +145,6 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme
</details>
## Egress / VPN Routing
These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `USING_WIREGUARD` | Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR. | boolean | `false` |
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick configuration file. | string (path) | `/config/wg0.conf` |
| `WIREGUARD_INTERFACE` | WireGuard interface name brought up by wg-quick. | string | `wg0` |
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable. | string (comma-separated) | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
| `WIREGUARD_ENFORCE_DNS` | Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable. | boolean | `true` |
| `WIREGUARD_DNS` | Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead. | string (comma-separated) | `unset (uses config DNS = line)` |
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface. | boolean | `true` |
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel. | boolean | `false` |
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK). | boolean | `false` |
| `WIREGUARD_STALE_AFTER` | Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel. | number | `180` |
<details>
<summary>Detailed descriptions</summary>
#### `USING_WIREGUARD`
Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.
- **Type:** boolean
- **Default:** `false`
#### `WIREGUARD_CONFIG`
Path to the mounted wg-quick configuration file.
- **Type:** string (path)
- **Default:** `/config/wg0.conf`
#### `WIREGUARD_INTERFACE`
WireGuard interface name brought up by wg-quick.
- **Type:** string
- **Default:** `wg0`
#### `LAN_NETWORK`
Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.
- **Type:** string (comma-separated)
- **Default:** `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`
#### `WIREGUARD_ENFORCE_DNS`
Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.
- **Type:** boolean
- **Default:** `true`
#### `WIREGUARD_DNS`
Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.
- **Type:** string (comma-separated)
- **Default:** `unset (uses config DNS = line)`
#### `WIREGUARD_DISABLE_IPV6`
Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.
- **Type:** boolean
- **Default:** `true`
#### `WIREGUARD_ALLOW_IPV6_LEAK`
Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.
- **Type:** boolean
- **Default:** `false`
#### `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL`
When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).
- **Type:** boolean
- **Default:** `false`
#### `WIREGUARD_STALE_AFTER`
Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.
- **Type:** number
- **Default:** `180`
</details>
## General
| Variable | Description | Type | Default |
@@ -317,7 +223,6 @@ Default language filter for searches.
| `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` |
| `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` |
| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` |
| `FORCE_COMBINED_SEARCH` | Force combined search whenever it's available. Locks the combined toggle on. | boolean | `false` |
| `METADATA_PROVIDER` | Choose which metadata provider to use for book searches. | string (choice) | `openlibrary` |
| `METADATA_PROVIDER_AUDIOBOOK` | Metadata provider for audiobook searches. Uses the book provider if not set. | string (choice) | _empty string_ |
| `METADATA_PROVIDER_COMBINED` | Metadata provider for combined mode searches. Uses the book provider if not set. | string (choice) | _empty string_ |
@@ -365,15 +270,6 @@ Show the option to search for and download both a book and audiobook together.
- **Type:** boolean
- **Default:** `true`
#### `FORCE_COMBINED_SEARCH`
**Always Use Combined Search**
Force combined search whenever it's available. Locks the combined toggle on.
- **Type:** boolean
- **Default:** `false`
#### `METADATA_PROVIDER`
**Book Metadata Provider**
@@ -433,8 +329,8 @@ The release source tab to open by default in the release modal for audiobooks. U
| `BOOKS_OUTPUT_MODE` | Choose where completed book files are sent. | string (choice) | `folder` |
| `INGEST_DIR` | Directory where downloaded files are saved. Use {User} for per-user folders (e.g. /books/{User}). | string | `/books` |
| `FILE_ORGANIZATION` | Choose how downloaded book files are named and organized. | string (choice) | `rename` |
| `TEMPLATE_RENAME` | Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title} ({Year})` |
| `TEMPLATE_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title} ({Year})` |
| `TEMPLATE_RENAME` | Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title} ({Year})` |
| `TEMPLATE_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title} ({Year})` |
| `HARDLINK_TORRENTS` | Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder. | boolean | `false` |
| `BOOKLORE_HOST` | Base URL of your Grimmory instance | string | _none_ |
| `BOOKLORE_USERNAME` | Grimmory account username | string | _none_ |
@@ -455,8 +351,8 @@ The release source tab to open by default in the release modal for audiobooks. U
| `EMAIL_ALLOW_UNVERIFIED_TLS` | Disable TLS certificate verification (not recommended). | boolean | `false` |
| `DESTINATION_AUDIOBOOK` | Directory where downloaded audiobook files are saved. Leave empty to use the Books destination. | string | _none_ |
| `FILE_ORGANIZATION_AUDIOBOOK` | Choose how downloaded audiobook files are named and organized. | string (choice) | `rename` |
| `TEMPLATE_AUDIOBOOK_RENAME` | Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title}` |
| `TEMPLATE_AUDIOBOOK_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title}/{Title}` |
| `TEMPLATE_AUDIOBOOK_RENAME` | Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title}` |
| `TEMPLATE_AUDIOBOOK_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title}/{Title}` |
| `HARDLINK_TORRENTS_AUDIOBOOK` | Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder. | boolean | `true` |
| `AUTO_OPEN_DOWNLOADS_SIDEBAR` | Automatically open the downloads sidebar when a new download is queued. | boolean | `false` |
| `DOWNLOAD_TO_BROWSER_CONTENT_TYPES` | Automatically download completed files to your browser for the selected content types. | string (comma-separated) | _empty list_ |
@@ -500,7 +396,7 @@ Choose how downloaded book files are named and organized.
**Naming Template**
Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads.
Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads.
- **Type:** string
- **Default:** `{Author} - {Title} ({Year})`
@@ -509,7 +405,7 @@ Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source
**Path Template**
Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty.
Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty.
- **Type:** string
- **Default:** `{Author}/{Title} ({Year})`
@@ -710,7 +606,7 @@ Choose how downloaded audiobook files are named and organized.
**Naming Template**
Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads.
Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads.
- **Type:** string
- **Default:** `{Author} - {Title}`
@@ -719,7 +615,7 @@ Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source
**Path Template**
Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty.
Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty.
- **Type:** string
- **Default:** `{Author}/{Title}/{Title}`
@@ -1047,13 +943,11 @@ Comma-separated hosts to bypass proxy (e.g., localhost,127.0.0.1,10.*,*.local)
|----------|-------------|------|---------|
| `URL_BASE` | Optional URL path prefix. Use a path like /shelfmark (no hostname). Leave blank for root. | string | _none_ |
| `DEBUG` | Enable verbose logging to console and file. Not recommended for normal use. | boolean | `false` |
| `LOG_LEVEL` | Lowest severity written to the console and log file. Ignored while Debug Mode is on, which forces Debug. | string (choice) | `INFO` |
| `MAIN_LOOP_SLEEP_TIME` | How often the download queue is checked for new items. | number | `5` |
| `DOWNLOAD_PROGRESS_UPDATE_INTERVAL` | How often download progress is broadcast to the UI. | number | `1` |
| `CUSTOM_SCRIPT` | Path to a script to run after each successful download. Must be executable. | string | _none_ |
| `CUSTOM_SCRIPT_PATH_MODE` | Pass the path to the custom script as an absolute path or relative to the destination folder. | string (choice) | `absolute` |
| `CUSTOM_SCRIPT_JSON_PAYLOAD` | Send a JSON payload to the script via stdin. Useful for multi-file imports (audiobooks) or richer metadata without relying on path parsing. | boolean | `false` |
| `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT` | How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows. | number | `60` |
| `COVERS_CACHE_ENABLED` | Cache book covers on the server for faster loading. | boolean | `true` |
| `COVERS_CACHE_TTL` | How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork). | number | `0` |
| `COVERS_CACHE_MAX_SIZE_MB` | Maximum disk space for cached covers. Oldest images are removed when limit is reached. | number | `500` |
@@ -1084,17 +978,6 @@ Enable verbose logging to console and file. Not recommended for normal use.
- **Default:** `false`
- **Requires restart:** Yes
#### `LOG_LEVEL`
**Log Level**
Lowest severity written to the console and log file. Ignored while Debug Mode is on, which forces Debug.
- **Type:** string (choice)
- **Default:** `INFO`
- **Requires restart:** Yes
- **Options:** `DEBUG` (Debug), `INFO` (Info), `WARNING` (Warning), `ERROR` (Error), `CRITICAL` (Critical)
#### `MAIN_LOOP_SLEEP_TIME`
**Queue Check Interval (seconds)**
@@ -1145,16 +1028,6 @@ Send a JSON payload to the script via stdin. Useful for multi-file imports (audi
- **Type:** boolean
- **Default:** `false`
#### `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT`
**Completed Path Wait (seconds)**
How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows.
- **Type:** number
- **Default:** `60`
- **Constraints:** min: 0, max: 3600
#### `COVERS_CACHE_ENABLED`
**Enable Cover Cache**
@@ -1224,7 +1097,6 @@ How long to cache individual book details. Default: 600 (10 minutes). Max: 60480
| `PROWLARR_API_KEY` | Found in Prowlarr: Settings > General > API Key | string (secret) | _none_ |
| `PROWLARR_INDEXERS` | Select which indexers to search. 📚 = has book categories. Leave empty to search all. | string (comma-separated) | _empty list_ |
| `PROWLARR_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
| `PROWLARR_COLLAPSE_DUPLICATES` | Collapse a release that several indexer entries returned down to a single row, keeping the entry with the best Prowlarr priority. Turn this off to see every entry that carried it, which is what makes results from filter-specific entries (freeleech and the like) visible. | boolean | `true` |
| `PROWLARR_USE_SEED_PREFERENCES` | Apply per-indexer seed time and ratio preferences from Prowlarr when sending torrents to the download client | boolean | `false` |
<details>
@@ -1277,15 +1149,6 @@ Automatically retry search without category filtering if no results are found
- **Type:** boolean
- **Default:** `false`
#### `PROWLARR_COLLAPSE_DUPLICATES`
**Show one row per release**
Collapse a release that several indexer entries returned down to a single row, keeping the entry with the best Prowlarr priority. Turn this off to see every entry that carried it, which is what makes results from filter-specific entries (freeleech and the like) visible.
- **Type:** boolean
- **Default:** `true`
#### `PROWLARR_USE_SEED_PREFERENCES`
**Use Prowlarr seed preferences**
@@ -1418,11 +1281,9 @@ Delay between requests in seconds to avoid rate limiting (0-10).
| `IRC_SERVER` | IRC server hostname | string | _none_ |
| `IRC_PORT` | IRC server port (usually 6697 for TLS, 6667 for plain) | number | `6697` |
| `IRC_USE_TLS` | Enable TLS/SSL encryption for the IRC connection. Disable for servers that don't support TLS. | boolean | `true` |
| `IRC_CHANNEL` | Channel name without the # prefix. Used for all searches unless a separate audiobook channel is configured below. | string | _none_ |
| `IRC_CHANNEL` | Channel name without the # prefix | string | _none_ |
| `IRC_NICK` | Your IRC nickname (required). Must be unique on the IRC network. | string | _none_ |
| `IRC_SEARCH_BOT` | The search bot to address queries to (required). Searches are sent as "@<bot> <query>". | string | _none_ |
| `IRC_AUDIOBOOK_CHANNEL` | Optional. Channel name (without the # prefix) to use for audiobook searches. Leave blank to use the main channel above for audiobooks too. | string | _none_ |
| `IRC_AUDIOBOOK_SEARCH_BOT` | Optional. Search bot for the audiobook channel. Leave blank to reuse the main search bot above. Only used when an audiobook channel is set. | string | _none_ |
| `IRC_SEARCH_BOT` | The search bot to query for results | string | _none_ |
| `IRC_CACHE_TTL` | How long to keep cached search results before they expire. | string (choice) | `2592000` |
<details>
@@ -1460,7 +1321,7 @@ Enable TLS/SSL encryption for the IRC connection. Disable for servers that don't
**Channel**
Channel name without the # prefix. Used for all searches unless a separate audiobook channel is configured below.
Channel name without the # prefix
- **Type:** string
- **Default:** _none_
@@ -1480,26 +1341,7 @@ Your IRC nickname (required). Must be unique on the IRC network.
**Search bot**
The search bot to address queries to (required). Searches are sent as "@<bot> <query>".
- **Type:** string
- **Default:** _none_
- **Required:** Yes
#### `IRC_AUDIOBOOK_CHANNEL`
**Audiobook channel**
Optional. Channel name (without the # prefix) to use for audiobook searches. Leave blank to use the main channel above for audiobooks too.
- **Type:** string
- **Default:** _none_
#### `IRC_AUDIOBOOK_SEARCH_BOT`
**Audiobook search bot**
Optional. Search bot for the audiobook channel. Leave blank to reuse the main search bot above. Only used when an audiobook channel is set.
The search bot to query for results
- **Type:** string
- **Default:** _none_
@@ -1521,12 +1363,9 @@ How long to keep cached search results before they expire.
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `PROWLARR_TORRENT_CLIENT` | Choose which torrent client to use | string (choice) | _empty string_ |
| `ALLDEBRID_API_KEY` | AllDebrid API Key (apiv4) from your AllDebrid account settings | string (secret) | _none_ |
| `REALDEBRID_API_KEY` | Real-Debrid API Key (Secret Token) from your Real-Debrid account settings | string (secret) | _none_ |
| `QBITTORRENT_URL` | Web UI URL of your qBittorrent instance | string | _none_ |
| `QBITTORRENT_USERNAME` | qBittorrent Web UI username | string | _none_ |
| `QBITTORRENT_PASSWORD` | qBittorrent Web UI password | string (secret) | _none_ |
| `QBITTORRENT_API_KEY` | Found in qBittorrent: Options > Web UI > API Key (qBittorrent 5.2.0+). Used instead of the username and password when set. | string (secret) | _none_ |
| `QBITTORRENT_CATEGORY` | Category to assign to book downloads in qBittorrent | string | `books` |
| `QBITTORRENT_CATEGORY_AUDIOBOOK` | Category for audiobook downloads. Leave empty to use the book category. | string | _empty string_ |
| `QBITTORRENT_DOWNLOAD_DIR` | Server-side directory where torrents are downloaded (optional, uses qBittorrent default if not specified) | string | _none_ |
@@ -1546,11 +1385,9 @@ How long to keep cached search results before they expire.
| `RTORRENT_URL` | XML-RPC URL of your rTorrent instance | string | _none_ |
| `RTORRENT_USERNAME` | HTTP Basic auth username (if authentication enabled) | string | _none_ |
| `RTORRENT_PASSWORD` | HTTP Basic auth password | string (secret) | _none_ |
| `RTORRENT_LABEL` | Label to assign to ebook downloads in rTorrent | string | `cwabd` |
| `RTORRENT_AUDIOBOOK_LABEL` | Label to assign to audiobook downloads in rTorrent (falls back to Book Label if not set) | string | _none_ |
| `RTORRENT_LABEL` | Label to assign to book downloads in rTorrent | string | `cwabd` |
| `RTORRENT_DOWNLOAD_DIR` | Server-side directory where torrents are downloaded (optional, uses rTorrent default if not specified) | string | _none_ |
| `PROWLARR_TORRENT_ACTION` | Choose whether to keep, remove, or move the torrent to another category or label after import | string (choice) | `keep` |
| `PROWLARR_TORRENT_POST_IMPORT_CATEGORY` | Category or label to assign after a successful import | string | _empty string_ |
| `PROWLARR_TORRENT_ACTION` | Remove deletes the torrent from your client immediately after import (stops seeding, files are kept); Keep leaves it in the client to continue seeding | string (choice) | `keep` |
| `PROWLARR_USENET_CLIENT` | Choose which usenet client to use | string (choice) | _empty string_ |
| `NZBGET_URL` | URL of your NZBGet instance | string | _none_ |
| `NZBGET_USERNAME` | NZBGet control username | string | `nzbget` |
@@ -1574,25 +1411,7 @@ Choose which torrent client to use
- **Type:** string (choice)
- **Default:** _empty string_
- **Options:** `""` (None), `alldebrid` (AllDebrid), `qbittorrent` (qBittorrent), `realdebrid` (Real-Debrid), `transmission` (Transmission), `deluge` (Deluge), `rtorrent` (rTorrent)
#### `ALLDEBRID_API_KEY`
**API Key**
AllDebrid API Key (apiv4) from your AllDebrid account settings
- **Type:** string (secret)
- **Default:** _none_
#### `REALDEBRID_API_KEY`
**API Key**
Real-Debrid API Key (Secret Token) from your Real-Debrid account settings
- **Type:** string (secret)
- **Default:** _none_
- **Options:** `""` (None), `qbittorrent` (qBittorrent), `transmission` (Transmission), `deluge` (Deluge), `rtorrent` (rTorrent)
#### `QBITTORRENT_URL`
@@ -1621,15 +1440,6 @@ qBittorrent Web UI password
- **Type:** string (secret)
- **Default:** _none_
#### `QBITTORRENT_API_KEY`
**API Key**
Found in qBittorrent: Options > Web UI > API Key (qBittorrent 5.2.0+). Used instead of the username and password when set.
- **Type:** string (secret)
- **Default:** _none_
#### `QBITTORRENT_CATEGORY`
**Book Category**
@@ -1805,20 +1615,11 @@ HTTP Basic auth password
**Book Label**
Label to assign to ebook downloads in rTorrent
Label to assign to book downloads in rTorrent
- **Type:** string
- **Default:** `cwabd`
#### `RTORRENT_AUDIOBOOK_LABEL`
**Audiobook Label**
Label to assign to audiobook downloads in rTorrent (falls back to Book Label if not set)
- **Type:** string
- **Default:** _none_
#### `RTORRENT_DOWNLOAD_DIR`
**Download Directory**
@@ -1832,20 +1633,11 @@ Server-side directory where torrents are downloaded (optional, uses rTorrent def
**Torrent Completion Action**
Choose whether to keep, remove, or move the torrent to another category or label after import
Remove deletes the torrent from your client immediately after import (stops seeding, files are kept); Keep leaves it in the client to continue seeding
- **Type:** string (choice)
- **Default:** `keep`
- **Options:** `keep` (Keep), `remove` (Remove), `change_category` (Change Category)
#### `PROWLARR_TORRENT_POST_IMPORT_CATEGORY`
**Post-Import Category**
Category or label to assign after a successful import
- **Type:** string
- **Default:** _empty string_
- **Options:** `keep` (Keep), `remove` (Remove)
#### `PROWLARR_USENET_CLIENT`
@@ -2097,26 +1889,6 @@ Default sort order for Google Books search results.
</details>
### Metadata Providers: Moly.hu
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `MOLY_ENABLED` | Enable Moly.hu as a metadata provider for book searches | boolean | `false` |
<details>
<summary>Detailed descriptions</summary>
#### `MOLY_ENABLED`
**Enable Moly.hu**
Enable Moly.hu as a metadata provider for book searches
- **Type:** boolean
- **Default:** `false`
</details>
## Direct Download
### Direct Download: Download Sources
@@ -2124,7 +1896,6 @@ Enable Moly.hu as a metadata provider for book searches
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `DIRECT_DOWNLOAD_ENABLED` | Show Direct Download in release-source lists and allow Direct mode searches. Add your own mirror URLs in the Mirrors tab before using it. | boolean | `false` |
| `DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH` | When language metadata is missing or unknown, parse the distant path (file path shown in search results) for language tags like [BD FR] or [En]. Also enables local language filtering so lgli files without AA language metadata are not excluded before the distant path can be checked. | boolean | `false` |
| `AA_DONATOR_KEY` | Enables fast download access on AA. Get this from your donator account page. | string (secret) | _none_ |
| `FAST_SOURCES_DISPLAY` | Always tried first, no waiting or bypass required. | JSON array | _see UI for defaults_ |
| `SOURCE_PRIORITY` | Fallback sources, may have waiting. Requires bypasser. Drag to reorder. | JSON array | _see UI for defaults_ |
@@ -2152,15 +1923,6 @@ Show Direct Download in release-source lists and allow Direct mode searches. Add
- **Type:** boolean
- **Default:** `false`
#### `DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH`
**Detect Language From Distant Path**
When language metadata is missing or unknown, parse the distant path (file path shown in search results) for language tags like [BD FR] or [En]. Also enables local language filtering so lgli files without AA language metadata are not excluded before the distant path can be checked.
- **Type:** boolean
- **Default:** `false`
#### `AA_DONATOR_KEY`
**Account Donator Key**
+3 -17
View File
@@ -12,7 +12,7 @@ With a subpath (`URL_BASE=/shelfmark/`):
https://<your-shelfmark-domain>/shelfmark/api/auth/oidc/callback
```
The callback URL is constructed from the incoming request, so your reverse proxy must forward `X-Forwarded-Proto` and `X-Forwarded-Host` correctly, including the external port when it is not the protocol default. PKCE (S256) is used automatically.
The callback URL is constructed from the incoming request, so your reverse proxy must forward `X-Forwarded-Proto` and `X-Forwarded-Host` correctly. PKCE (S256) is used automatically.
## Settings
@@ -30,19 +30,7 @@ Configure in **Settings → Security → Authentication Method → OIDC**.
| Auto-Provision Users | Create accounts on first login | `true` |
| Login Button Label | Custom text for the sign-in button | — |
Use **Test Connection** to verify discovery, client configuration, and the provider's token signing keys (JWKS) before attempting login.
> **Authentik users:** make sure your provider has a **Signing Key** selected (e.g. the default self-signed certificate). Without one, Authentik serves an empty JWKS document and every login fails with an OIDC callback error, even though the discovery document looks healthy.
## Account Linking
On login, Shelfmark matches the OIDC identity to a user account in this order:
1. **OIDC subject** — a user who has logged in through this provider before.
2. **Email** — a local account with the same (unique) email address. This only happens when the provider also asserts `email_verified: true` for the address; an unverified email would let anyone claim a local account by registering its address at the IdP.
3. Otherwise, a new account is created when **Auto-Provision Users** is enabled (username conflicts get a numeric suffix), or the login is rejected with "Account not found" when it is disabled.
If the `email_verified` claim is missing or `false`, email linking is silently skipped — a common surprise when the address was never verified at the identity provider (e.g. Keycloak's **Email verified** toggle on the user, or Authentik accounts created without email verification). Make sure the `email` scope is requested and the address is marked verified in your IdP.
Use **Test Connection** to verify discovery and client configuration before attempting login.
## Environment Variables
@@ -58,8 +46,6 @@ If `DISABLE_LOCAL_AUTH` and `OIDC_AUTO_REDIRECT` are both enabled, users are red
## Troubleshooting
- **No token signing keys (empty JWKS)** — The provider's JWKS endpoint returned no keys, so ID tokens can't be verified. In Authentik this happens when the provider has no **Signing Key** selected; pick one (e.g. the default self-signed certificate) and try again.
- **Issuer validation failed** — The issuer in the token doesn't match the discovery document. Check your provider's external URL / issuer configuration.
- **Callback URL mismatch** — Reverse proxy isn't forwarding `X-Forwarded-Proto` or `X-Forwarded-Host`, so the constructed callback URL doesn't match what's registered in the provider.
- **Account not found** — Auto-provision is disabled and the user hasn't been pre-created by an admin. If you pre-created the account with a matching email, see [Account Linking](#account-linking): the provider must send `email_verified: true` for linking to happen.
- **Login created a duplicate account instead of linking to my local one** — Email linking requires a verified email; see [Account Linking](#account-linking). With `DEBUG=true`, the log notes when linking is skipped because the address isn't verified.
- **Account not found** — Auto-provision is disabled and the user hasn't been pre-created by an admin.
+5 -7
View File
@@ -23,11 +23,10 @@ server {
location / {
proxy_pass http://shelfmark:8084;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
}
@@ -57,11 +56,11 @@ All Shelfmark paths (UI, API, assets, Socket.IO) are served under the base path.
location /shelfmark/ {
proxy_pass http://shelfmark:8084/shelfmark/;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_read_timeout 86400;
@@ -137,11 +136,11 @@ location /shelfmark/ {
proxy_pass http://shelfmark:8084/shelfmark/;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_read_timeout 86400;
@@ -159,7 +158,6 @@ If login, settings saves, or downloads appear to fail in the browser but the act
- Do not force `Connection: upgrade` on every request. That can break normal `POST` and `PUT` responses while the backend still processes them.
- If your proxy UI does not support conditional websocket headers, remove the forced websocket headers entirely and let Shelfmark fall back to polling.
- Keep the standard forwarded headers: `Host`, `X-Forwarded-For`, `X-Forwarded-Proto`, and `X-Forwarded-Host` when using a subpath or OIDC.
- Preserve the original port in `Host` and `X-Forwarded-Host` by using `$http_host` rather than `$host` when Shelfmark is exposed on a custom port.
This is especially relevant for Nginx Proxy Manager or custom advanced config snippets that add websocket headers globally.
+1 -8
View File
@@ -19,7 +19,7 @@ http://your-server:8084/?q=harry+potter
| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
| `format` | Filter by file format | `/?format=epub` |
| `content` | Filter by content type | `/?content=fiction` |
| `content_type` | Select media type (`ebook`, `audiobook`, or `combined`) in Universal mode only | `/?q=dune&content_type=audiobook` |
| `content_type` | Select media type (`ebook` or `audiobook`) in Universal mode only | `/?q=dune&content_type=audiobook` |
| `sort` | Sort order for results | `/?sort=newest` |
## Multiple Values
@@ -63,11 +63,6 @@ Some parameters support multiple values by repeating the parameter:
/?q=dune&content_type=audiobook
```
**Universal search forcing combined (ebook + audiobook):**
```
/?q=dune&content_type=combined
```
## Search Mode Behavior
### Direct Mode
@@ -79,8 +74,6 @@ When Search Mode is set to Direct, all parameters are used to filter results fro
`q`, `sort`, and `content_type` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
`content_type=combined` forces combined mode (search ebook and audiobook providers together), overriding the last-used preference. It is silently ignored if combined mode is unavailable (e.g. the combined selector is disabled in settings, or either content type is blocked by request policy).
## Notes
- URL parameters are read once on page load
+3 -45
View File
@@ -81,13 +81,6 @@ if is_truthy "$ENABLE_LOGGING_VALUE"; then
fi
fi
# Egress modes are mutually exclusive. Check this BEFORE starting either one so
# we never run tor.sh and then abort, leaving a half-configured network stack.
if [ "$USING_TOR" = "true" ] && [ "$USING_WIREGUARD" = "true" ]; then
echo "USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode." >&2
exit 1
fi
if [ "$USING_TOR" = "true" ]; then
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
echo "USING_TOR=true requires the container to start as root." >&2
@@ -97,15 +90,6 @@ if [ "$USING_TOR" = "true" ]; then
./tor.sh
fi
if [ "$USING_WIREGUARD" = "true" ]; then
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
echo "USING_WIREGUARD=true requires the container to start as root." >&2
echo "Non-root mode skips the privileged network setup WireGuard depends on." >&2
exit 1
fi
./wireguard.sh
fi
if [ "$FILE_LOGGING_ENABLED" = "true" ]; then
start_file_logging "$LOG_FILE"
fi
@@ -251,22 +235,13 @@ test_write() {
return 1
fi
# This is a probe: a failure here is expected (e.g. a fresh root-owned bind
# mount) and is recovered by the caller via change_ownership + re-probe. Hide
# the shell's "Permission denied"/"Read-only file system" stderr so a handled
# probe miss doesn't masquerade as a real boot failure in the logs.
if ! run_as_target_user sh -c 'echo 0123456789_TEST 2>/dev/null > "$1"' _ "$test_file"; then
if ! run_as_target_user sh -c 'echo 0123456789_TEST > "$1"' _ "$test_file"; then
echo "Failed to write test file in $folder as $USERNAME"
return 1
fi
FILE_CONTENT=$(cat "$test_file" 2>/dev/null || echo "")
# A folder can be writable but not deletable (e.g. a Synology share without
# "Delete subfolders and files"). That is not a boot failure - the app writes
# files in place on such shares - so don't let a failed cleanup print an
# alarming error or fail the probe.
run_as_target_user rm -f "$test_file" 2>/dev/null || \
echo "Note: could not remove test file in $folder (folder is writable but not deletable)"
rm -f "$test_file"
[ "$FILE_CONTENT" = "0123456789_TEST" ]
result=$?
if [ $result -eq 0 ]; then
@@ -473,29 +448,12 @@ else
if [ $config_ok -ne 0 ]; then
fail_unwritable_config_dir "$CONFIG_PATH"
fi
# The ingest/destination library (default /books) is user data and may be a
# bind mount owned by another uid; downloads fail with "Destination not
# writable" if the runtime user can't write there. Fix the top-level dir only
# (root mode) so we don't recursively chown a potentially huge library.
make_writable "${INGEST_DIR:-/books}" root
fi
# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
# upgrades work reliably on customer machines.
# Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
# Gunicorn rejects anything outside its own list, so normalize and fall back to
# info rather than letting a typo stop the container from booting.
if [ "$DEBUG" = "true" ]; then
gunicorn_loglevel=debug
else
gunicorn_loglevel=$(echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
[ "$gunicorn_loglevel" = "warn" ] && gunicorn_loglevel=warning
case "$gunicorn_loglevel" in
debug|info|warning|error|critical) ;;
*) gunicorn_loglevel=info ;;
esac
fi
gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
command="${GUNICORN_BIN} --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
# If DEBUG and not using an external bypass
+5 -5
View File
@@ -19,28 +19,28 @@ dependencies = [
"psutil",
"emoji",
"rarfile",
"qbittorrent-api>=2026.8.0",
"qbittorrent-api",
"transmission-rpc",
"authlib>=1.7.2,<1.8",
"apprise>=1.12.0",
"apprise>=1.10.0",
]
[project.optional-dependencies]
browser = [
"pyvirtualdisplay",
"pyautogui",
"seleniumbase==4.51.11",
"seleniumbase==4.48.4",
"python-xlib",
]
[dependency-groups]
dev = [
"basedpyright>=1.39.9",
"basedpyright>=1.39.3",
"prek",
"pytest",
"pytest-cov",
"pytest-xdist>=3.8.0",
"ruff==0.16.2",
"ruff==0.15.12",
"vulture>=2.14",
]
+1 -48
View File
@@ -2,9 +2,6 @@
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
> [!NOTE]
> This project is in a stable state as of May 2026 but is not under active maintenance.
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
Works great alongside the following library tools, with support for automatic imports:
@@ -44,7 +41,6 @@ Works great alongside the following library tools, with support for automatic im
### Prerequisites
- Docker & Docker Compose
- At least 2 GB of RAM available to the container when using the standard image — see [Memory Requirements](#memory-requirements)
### Installation
@@ -107,16 +103,6 @@ Environment variables work for initial setup and Docker deployments. They serve
| `PUID` / `PGID` | Runtime user/group for the default root-startup flow (also supports legacy `UID`/`GID`) | `1000` / `1000` |
| `SEARCH_MODE` | `direct` or `universal` | `universal` |
| `USING_TOR` | Enable Tor routing (requires root startup) | `false` |
| `USING_WIREGUARD` | Enable WireGuard VPN egress with kill-switch (requires root startup) | `false` |
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick config | `/config/wg0.conf` |
| `WIREGUARD_INTERFACE` | WireGuard interface name | `wg0` |
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI / internal clients stay reachable | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
| `WIREGUARD_ENFORCE_DNS` | Pin the resolver (via `WIREGUARD_DNS`, else the config's `DNS =`) so DNS can't silently fall back to an off-tunnel path. Designed for a trusted LAN resolver kept reachable via `LAN_NETWORK` (query leaves over the LAN; download still egresses via the tunnel) — it does **not** force queries through the tunnel. Docker's embedded resolver (`127.0.0.11`) is preserved when present so container-name resolution keeps working; pin its upstream via the container's `dns:` list. Fails closed if no resolver is available or `/etc/resolv.conf` is not writable. | `true` |
| `WIREGUARD_DNS` | Explicit resolver(s) to pin (comma/space separated). Use when the VPN's pushed DNS filters domains you need; point at a resolver reachable via the tunnel or an allowed LAN resolver. | _(unset; uses config `DNS =`)_ |
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 from the tunnel config (many container kernels lack the ip6tables `raw` table wg-quick needs) and remove IPv6 as a leak surface. | `true` |
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue even when an IPv6 kill-switch can't be installed AND IPv6 can't be disabled. Only set if the container has no IPv6 connectivity. | `false` |
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | Opt-in off-tunnel WebUI reachability. Default (`false`) keeps the kill-switch strictly fail-closed: the only off-tunnel egress is loopback, the tunnel device and the LAN allowlist. Set `true` only if a **non-LAN** client (e.g. a public reverse proxy on another segment) must reach the WebUI; it permits app-server **replies** (`--sport FLASK_PORT`, conntrack REPLY) off-tunnel — server replies only, never client-initiated egress. LAN clients never need it (covered by `LAN_NETWORK`). | `false` |
| `WIREGUARD_STALE_AFTER` | Seconds since the last handshake before the healthcheck bounces the tunnel. | `180` |
See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
@@ -139,17 +125,6 @@ docker compose up -d
The full-featured image with all network capabilities included.
#### Memory Requirements
The standard image ships a real Chromium browser, which it launches to solve Cloudflare challenges for Direct Download. Chromium needs room to run:
- **2 GB of RAM available to the container** is a safe minimum; 1 GB or less is where problems usually start
- Only relevant if you use Direct Download. Prowlarr, IRC and audiobook sources don't start the browser
When the container is starved of memory, Chromium fails to start and every Direct Download fails with unrelated-looking errors — repeated `403 detected; switching to bypasser` followed by `No download URL found`, and downloads that never complete. If you're seeing that, check the container's memory limit and the host's free memory before suspecting your ISP or DNS.
If you can't spare the memory, use the [Lite](#lite) image with an external resolver (e.g. FlareSolverr) running elsewhere.
#### Tor Routing
Optional Tor support for network privacy:
```bash
@@ -163,31 +138,12 @@ docker compose -f docker-compose.tor.yml up -d
- Timezone is auto-detected from Tor exit node
- Custom DNS/proxy settings are ignored when Tor is active
#### WireGuard VPN Routing
Optional WireGuard support to route all external egress through a VPN tunnel with a fail-closed kill-switch:
```bash
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.wireguard.yml
# place your wg-quick config where the compose mounts /config, as wg0.conf
docker compose -f docker-compose.wireguard.yml up -d
```
**Notes:**
- Requires root startup
- Requires `NET_ADMIN` and `NET_RAW` capabilities
- Mount a standard wg-quick config at `WIREGUARD_CONFIG` (default `/config/wg0.conf`)
- All non-LAN egress is forced through the tunnel; if the tunnel drops, external traffic **fails closed** while LAN ranges (WebUI, Prowlarr, qBittorrent) stay reachable
- IPv4 and IPv6 both fail closed. On kernels without a usable `ip6tables`, disable IPv6 for the container (`sysctls: net.ipv6.conf.all.disable_ipv6=1`, as in the compose example) or the container refuses to start rather than risk an IPv6 leak
- A supervised healthcheck bounces the tunnel if the handshake goes stale, and refreshes the endpoint allow rules so a roaming/rotated peer endpoint can reconnect
- Mutually exclusive with `USING_TOR`
- **DNS trust:** `WIREGUARD_DNS` must be a resolver you trust on a trusted network segment. When it is a LAN resolver (kept reachable off-tunnel by `LAN_NETWORK`), the query to that resolver leaves as plaintext UDP/53 on the LAN — the resolver is responsible for encrypting upstream. Two resolver paths exist: (1) when Docker's embedded resolver (`127.0.0.11`) is present it is **preserved** so container names (Prowlarr, qBittorrent) resolve — you MUST pin its upstream to a trusted resolver via the container's compose `dns:` list, since `WIREGUARD_DNS` cannot repoint the embedded resolver from inside the container; (2) otherwise `WIREGUARD_DNS`/the config `DNS =` line is written to `/etc/resolv.conf`. Setting `WIREGUARD_ENFORCE_DNS=false` is a **foot-gun**: with no embedded resolver present the container then uses its inherited resolver, which forwards to the Docker daemon's upstream **off-tunnel**, leaking your DNS. Leave enforcement on unless you have pinned the resolver another way.
### Lite
A lighter image without the built-in browser automation. Ideal for:
- **External services** - Already running FlareSolverr or similar for other applications
- **Alternative sources** - Using Prowlarr, IRC, or other configured sources
- **Audiobooks** - Using Shelfmark primarily for audiobooks
- **Constrained hosts** - No bundled browser, so it runs comfortably below the standard image's [memory requirements](#memory-requirements)
```bash
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
@@ -260,10 +216,7 @@ Logs are available via:
- `docker logs <container-name>`
- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
Log level is configurable under Settings → Advanced or via the `LOG_LEVEL` environment
variable (`DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`; case-insensitive, defaults to
`INFO`). The environment variable wins over the setting, and `DEBUG=true` forces `DEBUG`
regardless of either. Changes take effect on restart.
Log level is configurable via Settings or `LOG_LEVEL` environment variable.
## Development
-111
View File
@@ -238,113 +238,6 @@ def _generate_bootstrap_env_docs() -> list[str]:
return lines
def _generate_egress_env_docs() -> list[str]:
"""Generate documentation for VPN/Tor egress environment variables.
These are startup-only variables consumed by entrypoint.sh / wireguard.sh
(before and outside the settings registry) to select and configure the
transparent-egress kill-switch. `USING_TOR` has a registry-backed entry
under Network and is cross-referenced rather than repeated here so the two
mutually exclusive egress modes are discoverable side by side without
emitting a duplicate `#### USING_TOR` anchor.
"""
egress_vars = [
{
"name": "USING_WIREGUARD",
"description": "Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.",
"type": "boolean",
"default": "false",
},
{
"name": "WIREGUARD_CONFIG",
"description": "Path to the mounted wg-quick configuration file.",
"type": "string (path)",
"default": "/config/wg0.conf",
},
{
"name": "WIREGUARD_INTERFACE",
"description": "WireGuard interface name brought up by wg-quick.",
"type": "string",
"default": "wg0",
},
{
"name": "LAN_NETWORK",
"description": "Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.",
"type": "string (comma-separated)",
"default": "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
},
{
"name": "WIREGUARD_ENFORCE_DNS",
"description": "Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.",
"type": "boolean",
"default": "true",
},
{
"name": "WIREGUARD_DNS",
"description": "Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.",
"type": "string (comma-separated)",
"default": "unset (uses config DNS = line)",
},
{
"name": "WIREGUARD_DISABLE_IPV6",
"description": "Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.",
"type": "boolean",
"default": "true",
},
{
"name": "WIREGUARD_ALLOW_IPV6_LEAK",
"description": "Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.",
"type": "boolean",
"default": "false",
},
{
"name": "WIREGUARD_ALLOW_WEBUI_OFFTUNNEL",
"description": "When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).",
"type": "boolean",
"default": "false",
},
{
"name": "WIREGUARD_STALE_AFTER",
"description": "Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.",
"type": "number",
"default": "180",
},
]
lines = [
"## Egress / VPN Routing",
"",
"These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.",
"",
"| Variable | Description | Type | Default |",
"|----------|-------------|------|---------|",
]
lines.extend(
f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |"
for var in egress_vars
)
lines.append("")
lines.append("<details>")
lines.append("<summary>Detailed descriptions</summary>")
lines.append("")
for var in egress_vars:
lines.append(f"#### `{var['name']}`")
lines.append("")
lines.append(var["description"])
lines.append("")
lines.append(f"- **Type:** {var['type']}")
lines.append(f"- **Default:** `{var['default']}`")
lines.append("")
lines.append("</details>")
lines.append("")
return lines
def generate_env_docs() -> str:
"""Generate markdown documentation for all environment variables."""
# Import settings modules to ensure all settings are registered
@@ -389,7 +282,6 @@ def generate_env_docs() -> str:
# Generate TOC
toc_entries = [
"- [Bootstrap Configuration](#bootstrap-configuration)",
"- [Egress / VPN Routing](#egress--vpn-routing)",
]
# Ungrouped tabs first
@@ -415,9 +307,6 @@ def generate_env_docs() -> str:
# Add bootstrap environment variables documentation
lines.extend(_generate_bootstrap_env_docs())
# Add egress / VPN routing (startup-only, shell-driven) documentation
lines.extend(_generate_egress_env_docs())
# Generate documentation for ungrouped tabs
for tab in grouped_tabs.get(None, []):
lines.extend(_generate_tab_docs(tab))
-16
View File
@@ -47,22 +47,6 @@ def _coerce_timeout_ms(value: object, default: int) -> int:
return default
def max_duration_seconds() -> float:
"""Upper bound on how long get_bypassed_page() can take for one URL.
MAX_RETRY attempts at the configured read timeout, plus the exponential backoff waited
between them (jitter is < 1s per gap, counted as a full second to stay conservative).
Callers use this to declare a stall-detection grace; see shelfmark.download.activity.
"""
bypasser_timeout = _coerce_timeout_ms(config.get("EXT_BYPASSER_TIMEOUT", 60000), 60000)
read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
backoff_total = sum(
min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + 1.0
for attempt in range(1, MAX_RETRY)
)
return MAX_RETRY * read_timeout + backoff_total
def _fetch_via_bypasser(target_url: str) -> str | None:
"""Make a single request to the external bypasser service. Returns HTML or None."""
raw_bypasser_url = _coerce_config_str(
+3 -64
View File
@@ -50,9 +50,6 @@ _LOADING_BODY_LENGTH_MAX = 50
_PAGE_BODY_PREVIEW_CHARS = 500
_BROWSER_START_TIMEOUT_SECONDS = 45.0
_BYPASS_SUBPROCESS_TIMEOUT_SECONDS = 420.0
# Same budget as the Docker helper process, applied to the in-process CDP path so both
# branches of get() are bounded the same way.
_IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS
_BYPASS_CHILD_ENV = "SHELFMARK_INTERNAL_BYPASSER_CHILD"
# Challenge detection indicators
@@ -220,13 +217,7 @@ class _CdpWorker:
msg = "CDP worker loop not available"
raise RuntimeError(msg)
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
try:
return future.result(timeout=timeout)
except TimeoutError:
# Otherwise the coroutine keeps running in the worker loop after we stop
# waiting, holding the browser and racing the next bypass.
future.cancel()
raise
return future.result(timeout=timeout)
_CDP_WORKER = _CdpWorker()
@@ -905,11 +896,7 @@ def _run_bypass_in_current_process(url: str, retry: int, cancel_flag: Event | No
if os.environ.get(_BYPASS_CHILD_ENV) == "1":
return asyncio.run(_run_bypass())
# Bound the wait: this path runs in-process (non-Docker installs), holds the module-wide
# LOCKED for its whole duration, and neither page.get() nor page.wait() has a timeout of
# its own. Without a deadline here a single wedged CDP session blocks every subsequent
# bypass in the process forever.
return _CDP_WORKER.run(_run_bypass(), timeout=_IN_PROCESS_BYPASS_TIMEOUT_SECONDS)
return _CDP_WORKER.run(_run_bypass())
def _store_child_bypass_state(payload: dict[str, Any]) -> None:
@@ -952,17 +939,7 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
result_path = (
Path(tempfile.gettempdir()) / f"shelfmark-bypass-{os.getpid()}-{time.time_ns()}.json"
)
# DNS provider state lives only in the parent's memory (no disk persistence), so the
# freshly spawned helper would otherwise pre-resolve AA hostnames against the system
# resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
# active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
# resolver rules.
payload = {
"url": url,
"retry": retry,
"result_path": str(result_path),
"dns_config": network.get_dns_config(),
}
payload = {"url": url, "retry": retry, "result_path": str(result_path)}
env_vars = os.environ.copy()
env_vars[_BYPASS_CHILD_ENV] = "1"
env_vars = _prepare_child_browser_env(env_vars)
@@ -1269,16 +1246,6 @@ def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
return None
def max_duration_seconds() -> float:
"""Upper bound on how long get_bypassed_page() can take for one URL.
Both branches of get() are capped at _BYPASS_SUBPROCESS_TIMEOUT_SECONDS, and
get_bypassed_page() may call it twice (once, then again after a mirror/DNS rotation).
Callers use this to declare a stall-detection grace; see shelfmark.download.activity.
"""
return 2 * _BYPASS_SUBPROCESS_TIMEOUT_SECONDS
def get_bypassed_page(
url: str, selector: network.AAMirrorSelector | None = None, cancel_flag: Event | None = None
) -> str | None:
@@ -1311,30 +1278,6 @@ def get_bypassed_page(
return response_html
def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
"""Mirror the parent process's active DNS provider in this helper subprocess.
DNS state is in-memory only, so a fresh helper defaults to system DNS and would
pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
the same DoH/custom resolver the parent already validated.
"""
provider = str(dns_config.get("provider") or "").strip().lower()
# "auto" means the parent has not rotated off system DNS yet, so the helper's own
# default initialization already matches it - nothing to override.
if not provider or provider == "auto":
return
manual_servers = dns_config.get("servers") if provider == "manual" else None
try:
network.set_dns_provider(
provider,
manual_servers,
use_doh=bool(dns_config.get("doh_enabled")),
)
except (OSError, RuntimeError, ValueError) as exc:
logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
def _run_child_process() -> int:
"""CLI entrypoint used by the Docker helper subprocess."""
request = json.loads(sys.stdin.read() or "{}")
@@ -1344,10 +1287,6 @@ def _run_child_process() -> int:
request.get("retry"), _coerce_positive_int(app_config.MAX_RETRY, 10)
)
dns_config = request.get("dns_config")
if isinstance(dns_config, dict):
_apply_parent_dns_config(dns_config)
try:
html = get(url, retry=retry)
payload = {
+1 -2
View File
@@ -2,7 +2,6 @@
from __future__ import annotations
import hashlib
from typing import Any
from shelfmark.core.config import config
@@ -24,7 +23,7 @@ _BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
return f"{base_url}|{username}|{hashlib.sha256(password.encode()).hexdigest()}"
return f"{base_url}|{username}|{hash(password)}"
def _get_booklore_select_options(
+13 -57
View File
@@ -6,78 +6,34 @@ import shutil
import tempfile
from pathlib import Path
LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")
def string_to_bool(s: str) -> bool:
"""Convert string to boolean."""
return s.lower() in ["true", "yes", "1", "y"]
def _read_advanced_config(key: str) -> object | None:
"""Read a key from the advanced settings file (import-time safe)."""
config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
config_file = config_dir / "plugins" / "advanced.json"
if config_file.exists():
try:
with config_file.open() as f:
config = json.load(f)
if key in config:
return config[key]
except json.JSONDecodeError, OSError:
pass
return None
def _read_debug_from_config() -> bool:
"""Read DEBUG from env var or config file (import-time safe)."""
env_debug = os.environ.get("DEBUG")
if env_debug is not None:
return string_to_bool(env_debug)
value = _read_advanced_config("DEBUG")
if value is not None:
return bool(value)
# Try to read from config file
config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
config_file = config_dir / "plugins" / "advanced.json"
if config_file.exists():
try:
with config_file.open() as f:
config = json.load(f)
if "DEBUG" in config:
return bool(config["DEBUG"])
except json.JSONDecodeError, OSError:
pass
return False
def normalize_log_level(raw: str | None) -> str:
"""Normalize a log level name, falling back to INFO when unrecognized."""
if raw is None:
return "INFO"
normalized = raw.strip().upper()
# "WARN" is a logging alias, but gunicorn only accepts "warning".
if normalized == "WARN":
normalized = "WARNING"
if normalized not in LOG_LEVELS:
return "INFO"
return normalized
def _read_log_level_from_config(debug: bool) -> str:
"""Resolve the app log level from DEBUG, env var, or config file.
DEBUG wins when enabled, mirroring how entrypoint.sh picks gunicorn's level.
Otherwise LOG_LEVEL is read from the env var, then the settings file, and
falls back to INFO when unset or unrecognized.
"""
if debug:
return "DEBUG"
raw = os.environ.get("LOG_LEVEL")
if raw is None:
value = _read_advanced_config("LOG_LEVEL")
raw = value if isinstance(value, str) else None
return normalize_log_level(raw)
def _is_sqlite_file(path: Path) -> bool:
"""Check if a file is a valid SQLite database by reading magic bytes."""
try:
@@ -145,7 +101,7 @@ INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
# =============================================================================
DEBUG = _read_debug_from_config()
LOG_LEVEL = _read_log_level_from_config(DEBUG)
LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+1 -19
View File
@@ -115,7 +115,7 @@ def check_oidc_connection(
response.raise_for_status()
document = response.json()
required_fields = ["issuer", "authorization_endpoint", "token_endpoint", "jwks_uri"]
required_fields = ["issuer", "authorization_endpoint", "token_endpoint"]
missing_fields = [field for field in required_fields if field not in document]
if missing_fields:
return {
@@ -123,24 +123,6 @@ def check_oidc_connection(
"message": f"Discovery document missing fields: {', '.join(missing_fields)}",
}
# Logins verify the ID token against the provider's JWKS, so an empty key
# set (e.g. an Authentik provider with no Signing Key selected) means every
# login will fail even though discovery looks healthy.
jwks_uri = str(document["jwks_uri"])
jwks_response = requests.get(jwks_uri, timeout=10, verify=get_ssl_verify(jwks_uri))
jwks_response.raise_for_status()
jwks_document = jwks_response.json()
jwks_keys = jwks_document.get("keys") if isinstance(jwks_document, dict) else None
if not jwks_keys:
return {
"success": False,
"message": (
"Discovery document is valid, but the provider returned no token "
"signing keys (empty JWKS), so logins will fail. If you use "
"Authentik, select a Signing Key in the provider settings."
),
}
return {"success": True, "message": f"Connected to {document['issuer']}"}
except Exception as exc:
logger.exception("OIDC connection test failed")
+10 -92
View File
@@ -1,5 +1,6 @@
"""Core settings registration and derived configuration values."""
import json
from pathlib import Path
from typing import Any
@@ -14,7 +15,6 @@ from shelfmark.config.download_settings_handlers import (
check_books_destination,
)
from shelfmark.config.email_settings import check_email_connection
from shelfmark.core.languages import supported_book_languages
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
ActionButton,
@@ -36,9 +36,6 @@ from shelfmark.core.settings_registry import (
register_settings,
)
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
"""Validate advanced settings before persisting."""
@@ -46,40 +43,6 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
logger = setup_logger(__name__)
timeout_key = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
if timeout_key in values:
raw_timeout = values.get(timeout_key)
if isinstance(raw_timeout, bool):
return {
"error": True,
"message": "Completed Path Wait must be a number of seconds",
"values": values,
}
if raw_timeout is None:
return {
"error": True,
"message": "Completed Path Wait must be a number of seconds",
"values": values,
}
try:
timeout_seconds = int(raw_timeout)
except TypeError, ValueError:
return {
"error": True,
"message": "Completed Path Wait must be a number of seconds",
"values": values,
}
if timeout_seconds < 0 or timeout_seconds > _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX:
return {
"error": True,
"message": (
"Completed Path Wait must be between 0 and "
f"{_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX} seconds"
),
"values": values,
}
values[timeout_key] = timeout_seconds
mappings = values.get("PROWLARR_REMOTE_PATH_MAPPINGS")
if mappings is None:
return {"error": False, "values": values}
@@ -143,8 +106,11 @@ for key in ["CONFIG_DIR", "LOG_DIR", "TMP_DIR", "INGEST_DIR", "DEBUG", "DOCKERMO
if hasattr(env, key):
logger.debug(" %s: %s", key, getattr(env, key))
# Selectable book languages, without the resolution aliases clients do not need.
_SUPPORTED_BOOK_LANGUAGE = supported_book_languages()
# Load supported book languages from data file
# Path is relative to the package root, not this file
_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
with (_DATA_DIR / "book-languages.json").open() as file:
_SUPPORTED_BOOK_LANGUAGE = json.load(file)
# Directory settings
BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -487,14 +453,6 @@ def search_mode_settings() -> list[SettingsField]:
show_when={"field": "SEARCH_MODE", "value": "universal"},
user_overridable=True,
),
CheckboxField(
key="FORCE_COMBINED_SEARCH",
label="Always Use Combined Search",
description="Force combined search whenever it's available. Locks the combined toggle on.",
default=False,
show_when={"field": "SEARCH_MODE", "value": "universal"},
user_overridable=True,
),
HeadingField(
key="universal_mode_heading",
title="Universal Mode Settings",
@@ -1009,7 +967,7 @@ def download_settings() -> list[SettingsField]:
key="TEMPLATE_RENAME",
label="Naming Template",
description=(
"Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} "
"Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} "
"(source filename without extension). Universal adds: {Series}, "
"{SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: "
"{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. "
@@ -1028,7 +986,7 @@ def download_settings() -> list[SettingsField]:
key="TEMPLATE_ORGANIZE",
label="Path Template",
description=(
"Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, "
"Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, "
"{OriginalName} (source filename without extension). Universal adds: {Series}, "
"{SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: "
"{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty."
@@ -1291,7 +1249,7 @@ def download_settings() -> list[SettingsField]:
key="TEMPLATE_AUDIOBOOK_RENAME",
label="Naming Template",
description=(
"Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} "
"Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} "
"(source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, "
"{PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: "
"{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. "
@@ -1308,7 +1266,7 @@ def download_settings() -> list[SettingsField]:
key="TEMPLATE_AUDIOBOOK_ORGANIZE",
label="Path Template",
description=(
"Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, "
"Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, "
"{OriginalName} (source filename without extension), {Series}, {SeriesPosition}, "
"{Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: "
"{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty."
@@ -1482,17 +1440,6 @@ def download_source_settings() -> list[SettingsField]:
),
default=False,
),
CheckboxField(
key="DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH",
label="Detect Language From Distant Path",
description=(
"When language metadata is missing or unknown, parse the distant path "
"(file path shown in search results) for language tags like [BD FR] or [En]. "
"Also enables local language filtering so lgli files without AA language "
"metadata are not excluded before the distant path can be checked."
),
default=False,
),
PasswordField(
key="AA_DONATOR_KEY",
label="Account Donator Key",
@@ -1761,23 +1708,6 @@ def advanced_settings() -> list[SettingsField]:
default=False,
requires_restart=True,
),
SelectField(
key="LOG_LEVEL",
label="Log Level",
description=(
"Lowest severity written to the console and log file. "
"Ignored while Debug Mode is on, which forces Debug."
),
options=[
{"value": "DEBUG", "label": "Debug", "description": "Everything, very noisy."},
{"value": "INFO", "label": "Info", "description": "Normal activity (default)."},
{"value": "WARNING", "label": "Warning", "description": "Warnings and problems."},
{"value": "ERROR", "label": "Error", "description": "Failures only."},
{"value": "CRITICAL", "label": "Critical", "description": "Fatal errors only."},
],
default="INFO",
requires_restart=True,
),
NumberField(
key="MAIN_LOOP_SLEEP_TIME",
label="Queue Check Interval (seconds)",
@@ -1831,18 +1761,6 @@ def advanced_settings() -> list[SettingsField]:
title="Remote Path Mappings",
description="Map download client paths to paths inside Shelfmark. Needed when volume mounts differ between containers.",
),
NumberField(
key="DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT",
label="Completed Path Wait (seconds)",
description=(
"How long to wait after a torrent or usenet client reports completion "
"for the completed file path to become visible to Shelfmark. Increase "
"this for seedbox or remote-sync workflows."
),
default=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT,
min_value=0,
max_value=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX,
),
TableField(
key="PROWLARR_REMOTE_PATH_MAPPINGS",
label="Path Mappings",
-6
View File
@@ -82,7 +82,6 @@ _SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
"DEFAULT_RELEASE_SOURCE",
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
"SHOW_COMBINED_SELECTOR",
"FORCE_COMBINED_SEARCH",
*_SEARCH_PREFERENCE_PROVIDER_KEYS,
}
@@ -224,11 +223,6 @@ def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | N
return value, None
return bool(value), None
if key == "FORCE_COMBINED_SEARCH":
if isinstance(value, bool):
return value, None
return bool(value), None
return value, None
+1 -1
View File
@@ -149,7 +149,7 @@ def _find_existing_alias_user(
]
if not candidates:
return None
return min(candidates, key=lambda user: int(user.get("id") or 0), default=None)
return sorted(candidates, key=lambda user: int(user.get("id") or 0))[0]
def _resolve_create_username(
-138
View File
@@ -1,138 +0,0 @@
"""Canonical language resolution shared by every release source.
Release sources report a language in whatever shape their upstream uses: a
two-letter code, an ISO 639-2 three-letter code in either the bibliographic or
terminological form, or an English name. They all need the same ISO 639-1 code
out the other side, so the aliases live in one place (``data/book-languages.json``)
and adding a language means editing one file.
"""
import json
import threading
import unicodedata
from pathlib import Path
from shelfmark.core.logger import setup_logger
logger = setup_logger(__name__)
LANGUAGE_DATA_PATH = Path(__file__).resolve().parents[1].parent / "data" / "book-languages.json"
# Values a source uses to mean "we could not tell".
LANGUAGE_PLACEHOLDERS = frozenset({"", "-", "--", "unknown", "unk", "n/a", "na", "none", "null"})
_ALIAS_TO_CODE: dict[str, str] | None = None
_CODE_TO_NAME: dict[str, str] | None = None
_LOCK = threading.Lock()
# Separators that stand in for the hyphen in a subtag. The dashes turn up in
# codes copied from web pages -- "zh‑Hant" used U+2011, which renders close
# enough to both a hyphen and an underscore to go unnoticed -- and the
# underscore is the spelling Direct Download accepted before this module existed.
_SUBTAG_SEPARATORS = dict.fromkeys(map(ord, "‐‑‒–—―−﹘﹣-_"), "-")
def _fold(value: str) -> str:
"""Casefold, strip accents, and normalize subtag separators, so 'Español'
and 'espanol', or 'zh-Hant', 'zh‑Hant' and 'zh_Hant', all match."""
decomposed = unicodedata.normalize("NFKD", value).translate(_SUBTAG_SEPARATORS)
stripped = "".join(ch for ch in decomposed if not unicodedata.combining(ch))
return " ".join(stripped.split()).casefold()
def _load() -> tuple[dict[str, str], dict[str, str]]:
global _ALIAS_TO_CODE, _CODE_TO_NAME
if _ALIAS_TO_CODE is not None and _CODE_TO_NAME is not None:
return _ALIAS_TO_CODE, _CODE_TO_NAME
with _LOCK:
if _ALIAS_TO_CODE is not None and _CODE_TO_NAME is not None:
return _ALIAS_TO_CODE, _CODE_TO_NAME
alias_to_code: dict[str, str] = {}
code_to_name: dict[str, str] = {}
try:
raw = json.loads(LANGUAGE_DATA_PATH.read_text(encoding="utf-8"))
except OSError, ValueError:
logger.exception("Failed to load language data from %s", LANGUAGE_DATA_PATH)
raw = []
if not isinstance(raw, list):
logger.warning("Language data at %s is not a list", LANGUAGE_DATA_PATH)
raw = []
for item in raw:
if not isinstance(item, dict):
continue
code = str(item.get("code") or "").strip()
name = str(item.get("language") or "").strip()
if not code:
continue
code_to_name.setdefault(code, name or code)
for candidate in (code, name, *(item.get("aliases") or [])):
folded = _fold(str(candidate))
if folded and folded not in LANGUAGE_PLACEHOLDERS:
alias_to_code.setdefault(folded, code)
_ALIAS_TO_CODE = alias_to_code
_CODE_TO_NAME = code_to_name
return alias_to_code, code_to_name
def normalize_language(value: object) -> str | None:
"""Resolve any known spelling of a language to its ISO 639-1 code.
Accepts a two-letter code, an ISO 639-2 three-letter code in either the
bibliographic or terminological form, or an English name. Returns None for
anything unrecognised or for the placeholders a source uses to say it does
not know, so callers can treat "no language" uniformly.
"""
if value is None:
return None
folded = _fold(str(value))
if not folded or folded in LANGUAGE_PLACEHOLDERS:
return None
alias_to_code, _ = _load()
return alias_to_code.get(folded)
def language_name(code: str | None) -> str | None:
"""Return the English name for a language code, or None if unknown."""
if not code:
return None
_, code_to_name = _load()
return code_to_name.get(str(code).strip())
def language_alias_map() -> dict[str, str]:
"""Every known alias mapped to its code, for callers doing their own matching.
Direct Download scans free-text paths and needs the whole alias set up front
to look for, rather than resolving one candidate at a time.
"""
alias_to_code, _ = _load()
return dict(alias_to_code)
def supported_book_languages() -> list[dict[str, str]]:
"""The selectable languages, as ``{"language": ..., "code": ...}``.
Aliases are an implementation detail of resolution, so they are left out of
what the settings dropdown and the API hand to clients.
"""
_, code_to_name = _load()
return [{"language": name, "code": code} for code, name in code_to_name.items()]
def known_language_codes() -> frozenset[str]:
"""Every ISO 639-1 code the bundled language data defines."""
_, code_to_name = _load()
return frozenset(code_to_name)
-4
View File
@@ -108,9 +108,6 @@ class DownloadTask:
retry_expected_hash: str | None = None # Optional torrent hash used to match client downloads
retry_ratio_limit: float | None = None # Optional post-download seeding ratio
retry_seeding_time_limit_minutes: int | None = None # Optional post-download seeding time limit
retry_source_context: dict[str, Any] = field(
default_factory=dict
) # Source-private context for retry/re-resolution
can_retry_without_staged_source: bool = (
True # Whether the source can restart without a preserved staged file
)
@@ -119,7 +116,6 @@ class DownloadTask:
series_name: str | None = None
series_position: float | None = None # Float for novellas (e.g., 1.5)
subtitle: str | None = None # Book subtitle for naming templates
language: str | None = None # Release language code for the {Language} template variable
# Hardlinking support
original_download_path: str | None = None # Path in download client (for hardlinking)
+2 -30
View File
@@ -4,7 +4,6 @@ import re
from pathlib import Path
from typing import TYPE_CHECKING
from shelfmark.core.languages import LANGUAGE_PLACEHOLDERS, normalize_language
from shelfmark.core.logger import setup_logger
if TYPE_CHECKING:
@@ -20,7 +19,6 @@ KNOWN_TOKENS = [
"primarytitle",
"originalname",
"partnumber",
"language",
"subtitle",
"author",
"series",
@@ -68,33 +66,6 @@ def format_series_position(position: str | float | None) -> str:
return str(position)
def normalize_language_code(language: str | None) -> str:
"""Resolve a release language to the single spelling used in a path.
Sources report the same language in different shapes: "en", "eng", "English".
All of them have to collapse to one code, or the editions they identify end
up in separate folders, which is the collision this token exists to prevent.
Placeholder values render empty so `{ (Language)}` disappears entirely
rather than labelling a folder "(unknown)".
A language the bundled data does not know is kept, casefolded, rather than
dropped: it still separates editions, and it cannot collide with a resolved
code precisely because nothing resolves it.
"""
if not language:
return ""
resolved = normalize_language(language)
if resolved is not None:
return resolved
normalized = " ".join(str(language).split()).strip().casefold()
if normalized in LANGUAGE_PLACEHOLDERS:
return ""
return normalized
def derive_primary_title(title: str | None, subtitle: str | None) -> str:
"""Return the title without an explicit subtitle suffix when possible."""
title_value = " ".join(str(title or "").split()).strip()
@@ -120,7 +91,8 @@ PAD_NUMBERS_PATTERN = re.compile(r"\d+")
def natural_sort_key(path: str | Path) -> str:
"""Generate a sort key with padded numbers for natural sorting."""
return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), str(path).lower())
filename = Path(path).name.lower()
return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
def assign_part_numbers(
-42
View File
@@ -393,41 +393,6 @@ def _plugin_label(plugin: object, fallback_scheme: str) -> str:
return " ".join(parts)
def _apprise_proxy_env() -> dict[str, str]:
"""Build proxy env vars from app config so Apprise respects the proxy setting."""
import os
from shelfmark.core.config import config as _cfg
mode = str(_cfg.get("PROXY_MODE", "") or "").lower()
env: dict[str, str] = {}
if mode == "http":
http = str(_cfg.get("HTTP_PROXY", "") or "").strip()
https = str(_cfg.get("HTTPS_PROXY", "") or "").strip() or http
if http:
env["HTTP_PROXY"] = http
env["http_proxy"] = http
if https:
env["HTTPS_PROXY"] = https
env["https_proxy"] = https
elif mode == "socks5":
socks = str(_cfg.get("SOCKS5_PROXY", "") or "").strip()
if socks:
env["HTTP_PROXY"] = socks
env["http_proxy"] = socks
env["HTTPS_PROXY"] = socks
env["https_proxy"] = socks
no_proxy = str(_cfg.get("NO_PROXY", "") or "").strip()
if no_proxy and env:
env["NO_PROXY"] = no_proxy
env["no_proxy"] = no_proxy
# Don't override if the user already set these in the environment directly
return {k: v for k, v in env.items() if not os.environ.get(k)}
def _dispatch_to_apprise(
urls: Iterable[str],
*,
@@ -435,8 +400,6 @@ def _dispatch_to_apprise(
body: str,
notify_type: object,
) -> dict[str, Any]:
import os
normalized_urls = _normalize_urls(list(urls))
url_schemes = _extract_url_schemes(normalized_urls)
if not normalized_urls:
@@ -445,11 +408,6 @@ def _dispatch_to_apprise(
if apprise is None:
return {"success": False, "message": "Apprise is not installed"}
proxy_env = _apprise_proxy_env()
if proxy_env:
logger.debug("Applying proxy env for Apprise dispatch: %s", list(proxy_env.keys()))
os.environ.update(proxy_env)
valid_urls = 0
invalid_urls = 0
delivered_urls = 0
-33
View File
@@ -33,11 +33,6 @@ logger = setup_logger(__name__)
oauth = OAuth()
_RETURN_TO_SESSION_KEY = "oidc_return_to"
_OIDC_CLIENT_ERRORS = (OAuthError, OSError, RuntimeError, TypeError, ValueError)
_EMPTY_JWKS_MESSAGE = (
"Authentication failed: the identity provider returned no token signing keys "
"(empty JWKS). If you use Authentik, select a Signing Key in the provider "
"settings and try again."
)
class _ClaimsMappingLike(Protocol):
@@ -126,17 +121,6 @@ def _normalize_return_to(raw_return_to: object) -> str | None:
return urlunsplit(("", "", path, parsed.query, parsed.fragment))
def _idp_jwks_has_no_keys(client: Any) -> bool:
"""Return True when the IdP's JWKS document verifiably contains no signing keys."""
try:
jwk_set = client.fetch_jwk_set(force=True)
except (*_OIDC_CLIENT_ERRORS, KeyError):
return False
if not isinstance(jwk_set, Mapping):
return False
return not jwk_set.get("keys")
def _get_pending_return_to(*, clear: bool = False) -> str | None:
"""Read the pending post-login target from the session."""
raw_return_to = (
@@ -290,17 +274,6 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
return redirect(
_login_error_url(f"OIDC token claim validation failed: {claim_name}")
)
except KeyError, ValueError:
# An IdP serving an empty JWKS document (e.g. an Authentik provider
# with no Signing Key selected) surfaces as KeyError('keys') while
# importing the key set. Test Connection only validates discovery,
# so this is the first place the misconfiguration becomes visible.
if _idp_jwks_has_no_keys(client):
logger.exception(
"OIDC callback failed: the IdP JWKS document contains no signing keys"
)
return redirect(_login_error_url(_EMPTY_JWKS_MESSAGE))
raise
claims = _normalize_claims(token.get("userinfo"))
# If userinfo is missing or claims are too sparse, request it explicitly.
@@ -333,12 +306,6 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
is_admin = admin_group in groups
allow_email_link = bool(user_info.get("email")) and _is_email_verified(claims)
if user_info.get("email") and not allow_email_link:
logger.debug(
"OIDC email %s is not marked verified by the IdP; skipping "
"email-based account linking",
user_info["email"],
)
user = provision_oidc_user(
user_db,
user_info,
-4
View File
@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any
from werkzeug.utils import secure_filename
from shelfmark.config.env import normalize_log_level
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import coerce_bool, normalize_optional_text
@@ -899,9 +898,6 @@ def _get_env_value_for_field(field: FieldBase) -> tuple[bool, object | None]:
"WELIB_MIRROR_URLS",
} and isinstance(parsed, list):
parsed = _normalize_mirror_env_urls(parsed)
if field.key == "LOG_LEVEL" and isinstance(parsed, str):
# LOG_LEVEL is commonly set lowercase; the field options are uppercase.
parsed = normalize_log_level(parsed)
return True, parsed
if field.key == "AA_MIRROR_URLS":
-7
View File
@@ -52,13 +52,6 @@ def normalize_http_url(
if scheme:
normalized = f"{scheme}://{normalized}"
# Strip query string and fragment — mirrors are used as base URLs for
# constructing search requests; params/fragments on the configured URL
# produce malformed URLs when paths are appended (issue #999).
parsed = urlparse(normalized)
if parsed.query or parsed.fragment:
normalized = parsed._replace(query="", fragment="").geturl()
if strip_trailing_slash:
normalized = normalized.rstrip("/")
-81
View File
@@ -1,81 +0,0 @@
"""Stall-detection grace signalling for long single-shot download operations.
The orchestrator cancels a download after `STALL_TIMEOUT` seconds without activity, where
"activity" means a *changed* status event or a *changed* progress value. That de-duplication
is deliberate - a keep-alive that repeats the same payload on a timer proves nothing about
whether the operation is still making progress, so letting it refresh the stall clock would
make a genuinely wedged download immortal.
Operations that legitimately take longer than `STALL_TIMEOUT` but cannot report incremental
progress therefore declare an explicit upper bound up front instead:
request_activity_grace(status_callback, my_worst_case_seconds)
try:
...one long blocking call...
finally:
release_activity_grace(status_callback)
The grace is a single absolute deadline. It is never extended, so the operation still dies
if it overruns its own declared budget - just at *its* bound rather than at a global 300s.
The signal rides on the existing `status_callback` channel using a sentinel status, which
avoids threading a new parameter through every handler, post-processor and output module.
`shelfmark.download.orchestrator`'s per-task `status_callback` closure intercepts the
sentinel and never forwards it to `update_download_status`.
Adopters should be operations that yield to the gevent hub while blocking (`requests`,
patched `subprocess`). An operation that blocks the hub outright - `shutil.copy2`, sqlite -
will still be killed by the gunicorn worker timeout regardless of any grace, and must go
through `shelfmark.download.fs.run_blocking_io` first.
Current adopters: `shelfmark.download.http.html_get_page` (protection bypass).
Candidates: `download.clients.base_handler._wait_for_completed_path`, archive extraction in
`download.postprocess.scan`, large-file copies in `download.outputs.folder`, email/BookLore
uploads, and the Anna's Archive countdown in `release_sources.direct_download` (which today
refreshes the stall clock on every tick of a loop that proves nothing about the remote).
"""
from collections.abc import Callable
# Not a QueueStatus value, so `update_download_status` would reject it anyway; the
# orchestrator's status_callback intercepts it before that point.
ACTIVITY_GRACE_STATUS = "__activity_grace__"
StatusCallback = Callable[[str, str | None], None]
# A status_callback is caller-supplied and may raise; a failed liveness hint must never
# break the operation it was protecting. Mirrors http._STATUS_CALLBACK_ERRORS.
_CALLBACK_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError)
def request_activity_grace(status_callback: StatusCallback | None, seconds: float) -> None:
"""Ask the orchestrator to suppress stall detection for up to `seconds` from now."""
_emit(status_callback, seconds)
def release_activity_grace(status_callback: StatusCallback | None) -> None:
"""Drop any outstanding grace and count now as activity."""
_emit(status_callback, 0)
def parse_activity_grace(status: str, message: str | None) -> float | None:
"""Return the requested grace in seconds, or None if this is not a grace event.
Never raises: a malformed sentinel is treated as "not a grace event" so a bad emitter
cannot take down the status pipeline.
"""
if status != ACTIVITY_GRACE_STATUS:
return None
try:
return max(float(message or 0), 0.0)
except TypeError, ValueError:
return 0.0
def _emit(status_callback: StatusCallback | None, seconds: float) -> None:
if status_callback is None:
return
try:
status_callback(ACTIVITY_GRACE_STATUS, str(float(seconds)))
except _CALLBACK_ERRORS:
return
+2 -18
View File
@@ -325,19 +325,6 @@ class DownloadClient(ABC):
"""
def set_category(self, download_id: str, category: str) -> bool:
"""Update a download's category or label when supported by the client.
Args:
download_id: The client-specific download ID.
category: Category or label to assign.
Returns:
True if the category was updated, otherwise False.
"""
return False
@abstractmethod
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where files were downloaded.
@@ -372,13 +359,10 @@ class DownloadClient(ABC):
# Client registry: protocol -> list of client classes
_CLIENTS: dict[str, list[type[DownloadClient]]] = {}
ClientType = TypeVar("ClientType", bound=DownloadClient)
_BUILTIN_CLIENT_MODULES = (
"shelfmark.download.clients.alldebrid",
"shelfmark.download.clients.deluge",
"shelfmark.download.clients.nzbget",
"shelfmark.download.clients.qbittorrent",
"shelfmark.download.clients.realdebrid",
"shelfmark.download.clients.rtorrent",
"shelfmark.download.clients.sabnzbd",
"shelfmark.download.clients.transmission",
@@ -399,7 +383,7 @@ def _ensure_builtin_clients_registered() -> None:
def register_client(
protocol: str,
) -> Callable[[type[ClientType]], type[ClientType]]:
) -> Callable[[type[DownloadClient]], type[DownloadClient]]:
"""Register a download client for a protocol.
Multiple clients can be registered for the same protocol.
@@ -415,7 +399,7 @@ def register_client(
"""
def decorator(cls: type[ClientType]) -> type[ClientType]:
def decorator(cls: type[DownloadClient]) -> type[DownloadClient]:
if protocol not in _CLIENTS:
_CLIENTS[protocol] = []
_CLIENTS[protocol].append(cls)
-681
View File
@@ -1,681 +0,0 @@
"""AllDebrid debrid service client for Shelfmark.
Routes magnet links through the AllDebrid API (v4/v4.1) to download
torrent content via AllDebrid's CDN infrastructure.
"""
from __future__ import annotations
import shutil
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, ClassVar, NoReturn
from urllib.parse import quote
import requests
from shelfmark.config.env import TMP_DIR
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.download.clients import (
DownloadClient,
DownloadState,
DownloadStatus,
register_client,
)
from shelfmark.download.clients._coercion import config_text
from shelfmark.download.http import download_url
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_API_BASE = "https://api.alldebrid.com/v4"
_AGENT = "shelfmark"
_ALLDEBRID_CLIENT_ERRORS = (
AttributeError,
OSError,
requests.exceptions.RequestException,
RuntimeError,
TypeError,
ValueError,
)
# AllDebrid magnet status codes (from API v4.1 documentation).
_STATUS_DOWNLOADING = frozenset({0, 1, 2, 3})
_STATUS_READY = 4
# Timeouts and retry limits for API calls.
_API_TIMEOUT = 30
_STATUS_TIMEOUT = 15
_DELAYED_POLL_INTERVAL = 5
_DELAYED_POLL_MAX_ATTEMPTS = 12
# File extensions recognised as book or audiobook content.
_BOOK_EXTENSIONS = (
".aac",
".azw",
".azw3",
".cbr",
".cbz",
".djvu",
".doc",
".docx",
".epub",
".fb2",
".flac",
".lit",
".m4a",
".m4b",
".mobi",
".mp3",
".ogg",
".opus",
".pdf",
".rtf",
".txt",
".wma",
)
def _flatten_magnet_files(
entries: list[dict[str, Any]],
prefix: str = "",
) -> list[dict[str, Any]]:
"""Flatten AllDebrid's nested file tree into a list of file dicts.
AllDebrid returns files with ``"n"`` (name), ``"s"`` (size),
``"l"`` (link), and ``"e"`` (children) keys. Directories use
``"e"`` to nest their contents.
Returns:
List of ``{"filename": ..., "size": ..., "link": ...}`` dicts.
"""
flat: list[dict[str, Any]] = []
for entry in entries:
name = entry.get("n", "")
if "e" in entry:
flat.extend(
_flatten_magnet_files(entry["e"], prefix=f"{prefix}{name}/"),
)
elif entry.get("l"):
flat.append(
{
"filename": f"{prefix}{name}",
"size": entry.get("s", 0),
"link": entry["l"],
}
)
return flat
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
@dataclass
class _DownloadState:
"""Internal mutable state for an in-progress AllDebrid download."""
magnet_id: str
name: str
target_dir: Path
phase: str = "uploading"
error_message: str | None = None
progress: float = 0.0
download_thread: threading.Thread | None = None
lock: threading.Lock = field(default_factory=threading.Lock)
@register_client("torrent")
class AllDebridClient(DownloadClient):
"""AllDebrid debrid service client.
Downloads torrent content by uploading magnet links to AllDebrid,
waiting for the torrent to complete on their servers, then fetching
the resulting files via direct HTTP download from AllDebrid's CDN.
API documentation: https://docs.alldebrid.com/
"""
protocol = "torrent"
name = "alldebrid"
_downloads: ClassVar[dict[str, _DownloadState]] = {}
_downloads_lock = threading.Lock()
def __init__(self) -> None:
self._api_key = config_text(config.get("ALLDEBRID_API_KEY", ""))
def _auth_headers(self) -> dict[str, str]:
"""Return Authorization header dict for API requests."""
return {"Authorization": f"Bearer {self._api_key}"}
# ------------------------------------------------------------------
# DownloadClient interface
# ------------------------------------------------------------------
@staticmethod
def is_configured() -> bool:
"""Return True when AllDebrid is selected and an API key exists."""
client = config_text(config.get("PROWLARR_TORRENT_CLIENT", ""))
api_key = config_text(config.get("ALLDEBRID_API_KEY", ""))
return client == "alldebrid" and bool(api_key)
def test_connection(self) -> tuple[bool, str]:
"""Validate the API key and check Premium subscription status."""
if not self._api_key:
return False, "AllDebrid API Key is required"
try:
url = f"{_API_BASE}/user"
resp = requests.get(
url,
headers=self._auth_headers(),
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(url),
)
resp.raise_for_status()
data = resp.json()
if data.get("status") != "success":
err = data.get("error", {}).get("message", "API error")
return False, f"AllDebrid error: {err}"
user = data.get("data", {}).get("user", {})
username = user.get("username", "Unknown")
if not user.get("isPremium", False):
return (
False,
f"AllDebrid user '{username}' does not have a Premium subscription",
)
except _ALLDEBRID_CLIENT_ERRORS as e:
return False, f"Connection failed: {e}"
else:
return True, f"Connected to AllDebrid as '{username}' (Premium)"
def add_download(
self,
url: str,
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs: object,
) -> str:
"""Upload a magnet link to AllDebrid and return the magnet ID."""
if not self._api_key:
msg = "AllDebrid API key is not configured"
raise RuntimeError(msg)
magnet_link = url
if not magnet_link.startswith("magnet:") and expected_hash:
magnet_link = f"magnet:?xt=urn:btih:{expected_hash}"
api_url = f"{_API_BASE}/magnet/upload"
try:
resp = requests.post(
api_url,
headers=self._auth_headers(),
data={"magnets[]": magnet_link},
timeout=_API_TIMEOUT,
verify=get_ssl_verify(api_url),
)
resp.raise_for_status()
data = resp.json()
if data.get("status") != "success":
code = data.get("error", {}).get("code", "UNKNOWN")
msg = f"AllDebrid upload failed: {code}"
_raise_runtime_error(msg)
magnets = data.get("data", {}).get("magnets", [])
if not magnets:
msg = "No magnet returned from AllDebrid"
_raise_runtime_error(msg)
info = magnets[0]
if info.get("error"):
code = info["error"].get("code", "UNKNOWN")
msg = f"AllDebrid magnet error: {code}"
_raise_runtime_error(msg)
magnet_id = str(info.get("id", ""))
if not magnet_id:
msg = "No magnet ID returned from AllDebrid"
_raise_runtime_error(msg)
target_dir = TMP_DIR / f"alldebrid_{magnet_id}"
target_dir.mkdir(parents=True, exist_ok=True)
state = _DownloadState(
magnet_id=magnet_id,
name=name,
target_dir=target_dir,
phase="waiting_ad",
)
with self._downloads_lock:
self._downloads[magnet_id] = state
logger.info(
"Added torrent to AllDebrid: ID %s (%s)",
magnet_id,
name,
)
except Exception:
logger.exception("Failed to upload magnet to AllDebrid")
raise
else:
return magnet_id
def get_status(self, download_id: str) -> DownloadStatus:
"""Poll AllDebrid for magnet status and drive the download."""
state = self._ensure_state(download_id)
# Return cached terminal / in-flight states immediately.
with state.lock:
if state.phase == "error":
return DownloadStatus.error(
state.error_message or "AllDebrid error",
)
if state.phase == "complete":
return DownloadStatus(
progress=100.0,
state=DownloadState.COMPLETE,
message="Complete",
complete=True,
file_path=str(state.target_dir),
)
if state.phase == "downloading_http":
return DownloadStatus(
progress=state.progress,
state=DownloadState.DOWNLOADING,
message="Downloading files via HTTP...",
complete=False,
file_path=None,
)
# Ask AllDebrid for the current magnet status.
try:
status_url = f"{_API_BASE.replace('/v4', '/v4.1')}/magnet/status"
resp = requests.post(
status_url,
headers=self._auth_headers(),
data={"id": download_id},
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(status_url),
)
resp.raise_for_status()
data = resp.json()
if data.get("status") != "success":
err = data.get("error", {}).get("message", "Status failed")
return DownloadStatus.error(
f"AllDebrid status error: {err}",
)
mag = self._extract_magnet_info(data)
return self._handle_magnet_status(mag, state)
except Exception as e:
logger.exception(
"Error checking AllDebrid status for %s",
download_id,
)
return DownloadStatus.error(str(e))
def remove(
self,
download_id: str,
*,
delete_files: bool = False,
) -> bool:
"""Delete the magnet from AllDebrid and clean up local files."""
try:
url = f"{_API_BASE}/magnet/delete"
requests.post(
url,
headers=self._auth_headers(),
data={"id": download_id},
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(url),
)
except _ALLDEBRID_CLIENT_ERRORS as e:
logger.warning("Failed to delete magnet from AllDebrid: %s", e)
with self._downloads_lock:
state = self._downloads.pop(download_id, None)
if state and state.target_dir.exists():
shutil.rmtree(state.target_dir, ignore_errors=True)
return True
def get_download_path(self, download_id: str) -> str | None:
"""Return the local directory containing downloaded files."""
with self._downloads_lock:
state = self._downloads.get(download_id)
if state and state.phase == "complete":
return str(state.target_dir)
target_dir = TMP_DIR / f"alldebrid_{download_id}"
if target_dir.exists():
return str(target_dir)
return None
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _ensure_state(self, download_id: str) -> _DownloadState:
"""Get or create download state for the given magnet ID."""
with self._downloads_lock:
state = self._downloads.get(download_id)
if state:
return state
target_dir = TMP_DIR / f"alldebrid_{download_id}"
state = _DownloadState(
magnet_id=download_id,
name=f"Download {download_id}",
target_dir=target_dir,
phase="waiting_ad",
)
with self._downloads_lock:
self._downloads[download_id] = state
return state
@staticmethod
def _extract_magnet_info(data: dict[str, Any]) -> dict[str, Any]:
"""Extract magnet info dict from a status API response."""
mag_data = data.get("data", {}).get("magnets", {})
if isinstance(mag_data, list) and mag_data:
return mag_data[0]
if isinstance(mag_data, dict):
return mag_data
return {}
def _handle_magnet_status(
self,
mag: dict[str, Any],
state: _DownloadState,
) -> DownloadStatus:
"""Map AllDebrid magnet status to a DownloadStatus."""
status_code = mag.get("statusCode")
if status_code in _STATUS_DOWNLOADING:
size = mag.get("size", 0)
downloaded = mag.get("downloaded", 0)
pct = (downloaded / size * 100.0) if size > 0 else 0.0
return DownloadStatus(
progress=pct * 0.5,
state=DownloadState.DOWNLOADING,
message=(f"AllDebrid downloading torrent ({mag.get('filename', state.name)})"),
complete=False,
file_path=None,
download_speed=mag.get("downloadSpeed", 0),
)
if status_code == _STATUS_READY or mag.get("ready", False):
self._maybe_start_download_thread(state)
return DownloadStatus(
progress=50.0,
state=DownloadState.DOWNLOADING,
message="AllDebrid ready, retrieving files...",
complete=False,
file_path=None,
)
# Terminal error from AllDebrid.
error_txt = mag.get("error", {}).get("message") or f"AllDebrid status code {status_code}"
with state.lock:
state.phase = "error"
state.error_message = error_txt
return DownloadStatus.error(error_txt)
def _maybe_start_download_thread(self, state: _DownloadState) -> None:
"""Spawn a background thread to unlock and download files."""
with state.lock:
already_running = state.phase in (
"unlocking",
"downloading_http",
"complete",
)
thread_alive = state.download_thread is not None and state.download_thread.is_alive()
if already_running or thread_alive:
return
state.phase = "unlocking"
t = threading.Thread(
target=self._process_and_download,
args=(state,),
daemon=True,
)
state.download_thread = t
t.start()
# ------------------------------------------------------------------
# Link unlocking
# ------------------------------------------------------------------
def _unlock_file_link(self, link: str) -> str:
"""Resolve an AllDebrid file link to a direct CDN download URL.
AllDebrid's ``/v4/magnet/files`` endpoint returns virtual links
(``alldebrid.com/f/...``) that must be converted to direct CDN
URLs via ``/v4/link/unlock``.
Strategy:
1. If the link is already a CDN URL (``/dl/``), return it.
2. ``POST /v4/link/unlock`` with Bearer auth (primary).
3. ``GET /v4/link/unlock`` with query parameters (fallback).
4. Append ``apikey=`` to ``alldebrid.com/f/`` links
(last-resort fallback for ghost-cached torrents).
"""
# 1. Already a direct CDN link.
if "/dl/" in link:
return link
headers = self._auth_headers()
unlock_url = f"{_API_BASE}/link/unlock"
err_msg = "Unknown unlock error"
# 2. POST unlock (primary method).
try:
resp = requests.post(
unlock_url,
headers=headers,
data={"link": link},
timeout=_API_TIMEOUT,
verify=get_ssl_verify(unlock_url),
)
if resp.status_code == 200:
body = resp.json()
if body.get("status") == "success":
direct = self._resolve_unlock_data(
body.get("data", {}),
headers,
)
if direct:
return direct
err_msg = body.get("error", {}).get(
"message",
"Unlock failed",
)
except _ALLDEBRID_CLIENT_ERRORS as e:
logger.debug("POST unlock exception: %s", e)
# 3. GET unlock fallback with URL-encoded link.
try:
encoded = quote(link, safe="")
get_url = (
f"{_API_BASE}/link/unlock?agent={_AGENT}&apikey={self._api_key}&link={encoded}"
)
resp = requests.get(
get_url,
headers=headers,
timeout=_API_TIMEOUT,
verify=get_ssl_verify(get_url),
)
if resp.status_code == 200:
body = resp.json()
if body.get("status") == "success":
direct = body.get("data", {}).get("link")
if direct:
return direct
err_msg = body.get("error", {}).get("message", err_msg)
except _ALLDEBRID_CLIENT_ERRORS as e:
logger.debug("GET unlock exception: %s", e)
# 4. Last-resort: append apikey to alldebrid.com/f/ links.
if "alldebrid.com/f/" in link:
logger.info(
"Using apikey fallback for AllDebrid file link: %s",
link,
)
if "apikey=" not in link:
sep = "&" if "?" in link else "?"
return f"{link}{sep}apikey={self._api_key}"
return link
logger.error(
"AllDebrid unlock failed for '%s': %s",
link,
err_msg,
)
msg = f"AllDebrid unlock failed: {err_msg}"
raise RuntimeError(msg)
def _resolve_unlock_data(
self,
data: dict[str, Any],
headers: dict[str, str],
) -> str | None:
"""Extract the direct link from unlock response data.
Handles the *delayed link* flow where AllDebrid returns a
``delayed`` ID instead of an immediate download link.
"""
# Delayed link: poll until the CDN file is ready.
if "delayed" in data:
delayed_id = data["delayed"]
logger.info(
"AllDebrid link delayed (ID %s), polling...",
delayed_id,
)
delayed_url = f"{_API_BASE}/link/delayed"
for _ in range(_DELAYED_POLL_MAX_ATTEMPTS):
time.sleep(_DELAYED_POLL_INTERVAL)
try:
resp = requests.post(
delayed_url,
headers=headers,
data={"id": delayed_id},
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(delayed_url),
)
if resp.status_code != 200:
continue
body = resp.json()
d = body.get("data", {})
if body.get("status") == "success" and d.get("status") == 2 and d.get("link"):
return d["link"]
except _ALLDEBRID_CLIENT_ERRORS as e:
logger.debug("Delayed poll exception: %s", e)
return data.get("link")
# ------------------------------------------------------------------
# File download pipeline
# ------------------------------------------------------------------
def _process_and_download(self, state: _DownloadState) -> None:
"""Fetch the file list, unlock links, and download via HTTP.
Runs in a background thread spawned by ``_maybe_start_download_thread``.
"""
try:
files = self._fetch_file_list(state.magnet_id)
relevant = [f for f in files if f["filename"].lower().endswith(_BOOK_EXTENSIONS)]
if not relevant:
relevant = files
with state.lock:
state.phase = "downloading_http"
total = len(relevant)
for idx, file_info in enumerate(relevant):
direct_link = self._unlock_file_link(file_info["link"])
rel_path = Path(file_info["filename"])
dest = state.target_dir / rel_path
dest.parent.mkdir(parents=True, exist_ok=True)
logger.info(
"Downloading AllDebrid file %d/%d: %s",
idx + 1,
total,
rel_path,
)
buf = download_url(
direct_link,
referer="https://alldebrid.com/",
)
if not buf:
msg = f"Failed to download from {direct_link}"
_raise_runtime_error(msg)
with dest.open("wb") as fh:
fh.write(buf.getvalue())
with state.lock:
state.progress = 50.0 + (idx + 1) / total * 50.0
with state.lock:
state.phase = "complete"
state.progress = 100.0
logger.info(
"AllDebrid download complete for ID %s at %s",
state.magnet_id,
state.target_dir,
)
except Exception:
logger.exception(
"Error in AllDebrid download for ID %s",
state.magnet_id,
)
with state.lock:
state.phase = "error"
state.error_message = str(
state.error_message or "Download failed",
)
def _fetch_file_list(
self,
magnet_id: str,
) -> list[dict[str, Any]]:
"""Retrieve and flatten the file tree for a magnet."""
url = f"{_API_BASE}/magnet/files"
resp = requests.post(
url,
headers=self._auth_headers(),
data={"id[]": magnet_id},
timeout=_API_TIMEOUT,
verify=get_ssl_verify(url),
)
resp.raise_for_status()
data = resp.json()
if data.get("status") != "success":
msg = f"Failed to list magnet files: {data.get('error')}"
raise RuntimeError(msg)
magnets = data.get("data", {}).get("magnets", [])
if not magnets:
msg = "No magnet files returned"
raise RuntimeError(msg)
files = _flatten_magnet_files(magnets[0].get("files", []))
if not files:
msg = "No files found in torrent"
raise RuntimeError(msg)
return files
+29 -179
View File
@@ -3,7 +3,6 @@
from __future__ import annotations
import errno
import math
import shutil
import time
from abc import ABC, abstractmethod
@@ -56,19 +55,6 @@ SECONDS_PER_HOUR = 3600
# How long to wait for completed files to appear (seconds)
COMPLETED_PATH_RETRY_INTERVAL = 5
COMPLETED_PATH_MAX_ATTEMPTS = 12 # 12 attempts * 5s = 60s grace period
COMPLETED_PATH_TIMEOUT_SETTING = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
COMPLETED_PATH_TIMEOUT_MAX_SECONDS = 3600
_RETRYABLE_COMPLETED_PATH_ERRNOS = frozenset(
code
for code in (
errno.ENOENT,
getattr(errno, "ESTALE", None),
getattr(errno, "EAGAIN", None),
getattr(errno, "EBUSY", None),
getattr(errno, "ETIMEDOUT", None),
)
if code is not None
)
@dataclass(frozen=True)
@@ -83,39 +69,6 @@ class DownloadRequest:
ratio_limit: float | None = None
@dataclass(frozen=True)
class _CompletedPathResolution:
path: Path | None
error: str | None
retryable: bool
def _coerce_completed_path_timeout_seconds(value: object, default: float) -> float:
if isinstance(value, bool) or value is None:
return default
if isinstance(value, (int, float)):
parsed = float(value)
elif isinstance(value, str):
try:
parsed = float(value.strip())
except ValueError:
return default
else:
return default
if not math.isfinite(parsed) or parsed < 0:
return default
return min(parsed, float(COMPLETED_PATH_TIMEOUT_MAX_SECONDS))
def _is_retryable_completed_path_probe(error: OSError | None) -> bool:
return error is not None and error.errno in _RETRYABLE_COMPLETED_PATH_ERRNOS
def _path_needs_mapping(path: str) -> bool:
return (len(path) >= WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":") or "\\" in path
def _diagnose_path_issue(path: str) -> str:
"""Analyze a path and return diagnostic hints for common issues.
@@ -214,23 +167,6 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Maximum attempts when waiting for completed files."""
return COMPLETED_PATH_MAX_ATTEMPTS
def _completed_path_timeout_seconds(self) -> float:
"""Total time to wait for completed files to appear on disk."""
fallback = self._completed_path_retry_interval() * self._completed_path_max_attempts()
configured = config.get(COMPLETED_PATH_TIMEOUT_SETTING, fallback)
return _coerce_completed_path_timeout_seconds(configured, fallback)
def _refresh_download_request_after_add_failure(
self,
*,
task: DownloadTask,
request: DownloadRequest,
error: Exception,
status_callback: Callable[[str, str | None], None],
) -> DownloadRequest | None:
"""Give source handlers one chance to refresh stale resolved download data."""
return None
def _get_category_for_task(self, client: DownloadClient, task: DownloadTask) -> str | None:
"""Get audiobook category if configured and applicable, else None for default."""
if not is_audiobook(task.content_type):
@@ -284,47 +220,17 @@ class ExternalClientHandler(DownloadHandler, ABC):
)
elif protocol == "torrent":
torrent_action = config.get("PROWLARR_TORRENT_ACTION", "keep")
if torrent_action == "remove":
try:
client.remove(download_id, delete_files=False)
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to remove torrent %s from %s: %s",
download_id,
getattr(client, "name", "client"),
e,
)
if config.get("PROWLARR_TORRENT_ACTION", "keep") != "remove":
return
if torrent_action != "change_category":
return
post_import_category = normalize_optional_text(
config.get("PROWLARR_TORRENT_POST_IMPORT_CATEGORY", "")
)
if post_import_category is None:
return
try:
category_updated = client.set_category(download_id, post_import_category)
client.remove(download_id, delete_files=False)
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to set post-import category for torrent %s in %s: %s",
"Failed to remove torrent %s from %s: %s",
download_id,
getattr(client, "name", "client"),
e,
)
return
if not category_updated:
# Clients that cannot label torrents (debrid services) return False here,
# and the ones that can already log the specific failure themselves.
logger.debug(
"Post-import category not applied to torrent %s in %s",
download_id,
getattr(client, "name", "client"),
)
def _remove_usenet_download(
self,
@@ -481,21 +387,6 @@ class ExternalClientHandler(DownloadHandler, ABC):
log_details: bool,
) -> tuple[Path | None, str | None]:
"""Resolve and validate the completed download path once."""
result = self._resolve_download_path_once_detailed(
client,
download_id,
log_details=log_details,
)
return result.path, result.error
def _resolve_download_path_once_detailed(
self,
client: DownloadClient,
download_id: str,
*,
log_details: bool,
) -> _CompletedPathResolution:
"""Resolve and validate a completed path, including retryability."""
try:
raw_path = client.get_download_path(download_id)
except Exception as e:
@@ -511,7 +402,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.debug(
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
)
return _CompletedPathResolution(None, message, retryable=False)
return None, message
if not raw_path:
message = (
@@ -526,7 +417,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.debug(
"Download client returned empty path for %s %s", client.name, download_id
)
return _CompletedPathResolution(None, message, retryable=False)
return None, message
from shelfmark.core.path_mappings import (
get_client_host_identifier,
@@ -566,7 +457,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.error(failure_log, *failure_args)
else:
logger.debug(failure_log, *failure_args)
return _CompletedPathResolution(None, message, retryable=False)
return None, message
remapped_exists, remapped_error = _probe_completed_path(remapped)
@@ -589,7 +480,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
source_path_obj,
remapped,
)
return _CompletedPathResolution(remapped, None, retryable=False)
return remapped, None
message = (
f"Remapped path '{remapped}' does not exist. "
@@ -608,11 +499,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.error(failure_log, *failure_args)
else:
logger.debug(failure_log, *failure_args)
return _CompletedPathResolution(
None,
message,
retryable=_is_retryable_completed_path_probe(remapped_error),
)
return None, message
source_exists, source_error = _probe_completed_path(source_path_obj)
@@ -635,7 +522,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
download_id,
source_path_obj,
)
return _CompletedPathResolution(source_path_obj, None, retryable=False)
return source_path_obj, None
hint = _diagnose_path_issue(raw_path)
if mappings:
@@ -668,12 +555,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.error(failure_log, *failure_args)
else:
logger.debug(failure_log, *failure_args)
return _CompletedPathResolution(
None,
message,
retryable=not _path_needs_mapping(raw_path)
and _is_retryable_completed_path_probe(source_error),
)
return None, message
def _wait_for_completed_path(
self,
@@ -685,37 +567,23 @@ class ExternalClientHandler(DownloadHandler, ABC):
) -> tuple[Path | None, str | None]:
"""Wait briefly for completed files to appear on disk."""
last_error: str | None = None
max_attempts = self._completed_path_max_attempts()
retry_interval = self._completed_path_retry_interval()
timeout_seconds = self._completed_path_timeout_seconds()
if retry_interval <= 0 or timeout_seconds <= 0:
max_attempts = 1
else:
max_attempts = int(math.ceil(timeout_seconds / retry_interval)) + 1
for attempt in range(1, max_attempts + 1):
if cancel_flag and cancel_flag.is_set():
return None, last_error
log_details = attempt == max_attempts
result = self._resolve_download_path_once_detailed(
resolved_path, error = self._resolve_download_path_once(
client,
download_id,
log_details=log_details,
)
if result.path:
return result.path, None
if resolved_path:
return resolved_path, None
last_error = result.error
if not result.retryable:
if not log_details:
logger.error(
"Completed path resolution is not retryable for %s (%s): %s",
client.name,
download_id,
last_error,
)
return None, last_error
last_error = error
if attempt < max_attempts:
status_callback("locating", "Waiting for completed files...")
@@ -832,38 +700,20 @@ class ExternalClientHandler(DownloadHandler, ABC):
status_callback("downloading", "Resuming existing download")
else:
# No existing download - add new
refresh_attempted = False
while True:
status_callback("resolving", f"Sending to {client.name}")
try:
download_id = client.add_download(
url=request.url,
name=request.release_name,
category=category,
expected_hash=request.expected_hash,
seeding_time_limit=request.seeding_time_limit,
ratio_limit=request.ratio_limit,
)
except Exception as e:
if not refresh_attempted:
refresh_attempted = True
refreshed_request = self._refresh_download_request_after_add_failure(
task=task,
request=request,
error=e,
status_callback=status_callback,
)
if (
refreshed_request is not None
and refreshed_request.protocol == request.protocol
):
request = refreshed_request
continue
logger.exception("Failed to add to %s", client.name)
status_callback("error", f"Failed to add to {client.name}: {e}")
return None
break
status_callback("resolving", f"Sending to {client.name}")
try:
download_id = client.add_download(
url=request.url,
name=request.release_name,
category=category,
expected_hash=request.expected_hash,
seeding_time_limit=request.seeding_time_limit,
ratio_limit=request.ratio_limit,
)
except Exception as e:
logger.exception("Failed to add to %s", client.name)
status_callback("error", f"Failed to add to {client.name}: {e}")
return None
logger.info(
"Added to %s: %s for '%s'", client.name, download_id, request.release_name
+3 -18
View File
@@ -227,10 +227,10 @@ class DelugeClient(DownloadClient):
return self._rpc_call("daemon.info")
def _try_set_label(self, torrent_id: str, label: str) -> bool:
def _try_set_label(self, torrent_id: str, label: str) -> None:
"""Best-effort label assignment (requires Deluge Label plugin)."""
if not label:
return False
return
try:
# label.add will error if the plugin is unavailable or the label exists.
@@ -240,9 +240,6 @@ class DelugeClient(DownloadClient):
self._rpc_call("label.set_torrent", torrent_id, label)
except _DELUGE_CLIENT_ERRORS as e:
logger.debug("Could not set Deluge label '%s' for %s: %s", label, torrent_id, e)
return False
else:
return True
@staticmethod
def is_configured() -> bool:
@@ -280,10 +277,7 @@ class DelugeClient(DownloadClient):
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
if not torrent_info.is_magnet and not torrent_info.torrent_data:
message = "Failed to fetch torrent file"
if torrent_info.fetch_error:
message = f"{message}: {torrent_info.fetch_error}"
_raise_runtime_error(message)
_raise_runtime_error("Failed to fetch torrent file")
options: dict[str, Any] = {}
if self._download_dir:
@@ -425,15 +419,6 @@ class DelugeClient(DownloadClient):
else:
return False
def set_category(self, download_id: str, category: str) -> bool:
"""Assign a label to a torrent using Deluge's Label plugin."""
try:
self._ensure_connected()
return self._try_set_label(download_id, category)
except _DELUGE_CLIENT_ERRORS as e:
self._log_error("set_category", e)
return False
def get_download_path(self, download_id: str) -> str | None:
"""Return the resolved download path for a Deluge torrent."""
try:
+179 -206
View File
@@ -44,7 +44,6 @@ _HASH_LENGTH_40 = 40
_HASH_LENGTH_ED2K = 32
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
_METADATA_DOWNLOAD_STATES = {"forcedMetaDL", "metaDL"}
_ONE_WEEK_IN_SECONDS = 604800
@@ -95,25 +94,6 @@ def _hashes_match(hash1: str, hash2: str) -> bool:
return False
def _torrent_matches_download_id(torrent: object, download_id: str) -> bool:
"""Match an ID against every identity qBittorrent exposes.
For hybrid torrents, qBittorrent's primary `hash` can change from the v1
hash to the truncated v2 hash after metadata resolution. The full
`infohash_v1` and `infohash_v2` fields preserve the torrent's identities.
"""
identifiers = (
getattr(torrent, "hash", None),
getattr(torrent, "infohash_v1", None),
getattr(torrent, "infohash_v2", None),
)
return any(
isinstance(identifier, str) and identifier and _hashes_match(identifier, download_id)
for identifier in identifiers
)
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
@@ -186,6 +166,63 @@ def _build_qbittorrent_child_path(base_path: object, child_path: object) -> str
class QBittorrentClient(DownloadClient):
"""qBittorrent download client."""
def _is_torrent_loaded(self, torrent_hash: str) -> tuple[bool, str | None]:
"""Check whether qBittorrent has registered a torrent yet.
Uses `/api/v2/torrents/properties?hash=<hash>`.
Returns:
(loaded, error_message)
Notes:
A false result with no error means "not loaded yet".
"""
url = f"{self._base_url}/api/v2/torrents/properties"
params = {"hash": torrent_hash}
try:
self._client.auth_log_in()
response = self._client._session.get(url, params=params, timeout=10)
# Re-authenticate and retry once on 403
if response.status_code == _HTTP_STATUS_FORBIDDEN:
logger.debug(
"qBittorrent returned 403 for properties; re-authenticating and retrying"
)
self._client.auth_log_in()
response = self._client._session.get(url, params=params, timeout=10)
if response.status_code == _HTTP_STATUS_FORBIDDEN:
return False, "qBittorrent authentication failed (HTTP 403)"
# qBittorrent returns 404/409-ish responses depending on version when missing.
if response.status_code == _HTTP_STATUS_NOT_FOUND:
return False, None
response.raise_for_status()
except requests.exceptions.HTTPError as e:
status = getattr(getattr(e, "response", None), "status_code", None)
if status == _HTTP_STATUS_NOT_FOUND:
return False, None
if status:
return False, f"qBittorrent API request failed (HTTP {status})"
return False, "qBittorrent API request failed"
except requests.exceptions.ConnectionError:
return False, f"Cannot connect to qBittorrent at {self._base_url}"
except requests.exceptions.Timeout:
return False, f"qBittorrent request timed out at {self._base_url}"
except requests.exceptions.InvalidSchema:
return (
False,
"qBittorrent URL is invalid (missing http:// or https://). "
f"Configured: {self._base_url}",
)
except _QBITTORRENT_CLIENT_ERRORS as e:
return False, f"qBittorrent API error: {type(e).__name__}: {e}"
else:
return True, None
protocol = "torrent"
name = "qbittorrent"
@@ -207,7 +244,6 @@ class QBittorrentClient(DownloadClient):
username = config_text(config.get("QBITTORRENT_USERNAME", ""))
password = config_text(config.get("QBITTORRENT_PASSWORD", ""))
self._api_key = config_text(config.get("QBITTORRENT_API_KEY", ""))
# qbittorrent-api accepts either a full URL or host:port; prefer the normalized URL
# for consistency.
@@ -215,40 +251,43 @@ class QBittorrentClient(DownloadClient):
host=self._base_url,
username=username,
password=password,
api_key=self._api_key or None,
VERIFY_WEBUI_CERTIFICATE=get_ssl_verify(self._base_url),
)
self._category = config_text(config.get("QBITTORRENT_CATEGORY", "books"))
self._download_dir = config_text(config.get("QBITTORRENT_DOWNLOAD_DIR", ""))
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
@property
def _can_reauthenticate(self) -> bool:
"""Whether a 403 is worth retrying; a bearer token cannot be refreshed like a session."""
return not self._api_key
def _ensure_authenticated(self) -> None:
"""Authenticate the underlying HTTP session before it is used directly.
API keys (qBittorrent 5.2.0+) are sent as a bearer header on every request and
have no login endpoint, so there is no session to establish up front.
"""
if self._api_key:
return
self._client.auth_log_in()
def _request_torrent_info_records(
self, params: dict[str, str]
def _get_torrents_info(
self, torrent_hash: str | None = None
) -> tuple[list[SimpleNamespace], str | None]:
"""Request torrent info records from qBittorrent."""
"""Get torrent info using GET.
Behaviors:
- Retry once on HTTP 403 by re-authenticating.
- Keep "API/auth/connect" errors distinct from "torrent missing".
- If a hash-specific query returns empty, fall back to listing by category
and matching locally.
Returns:
(torrents, error_message)
"""
url = f"{self._base_url}/api/v2/torrents/info"
try:
self._ensure_authenticated()
response = self._client._session.get(url, params=params, timeout=10)
if response.status_code == _HTTP_STATUS_FORBIDDEN and self._can_reauthenticate:
def do_request(params: dict[str, str]) -> requests.Response:
# Ensure session is authenticated before using it directly
self._client.auth_log_in()
return self._client._session.get(url, params=params, timeout=10)
def parse_response(
response: requests.Response,
*,
request_params: dict[str, str],
) -> tuple[list[SimpleNamespace], str | None]:
if response.status_code == _HTTP_STATUS_FORBIDDEN:
logger.debug("qBittorrent returned 403; re-authenticating and retrying")
self._ensure_authenticated()
response = self._client._session.get(url, params=params, timeout=10)
self._client.auth_log_in()
response = self._client._session.get(url, params=request_params, timeout=10)
if response.status_code == _HTTP_STATUS_FORBIDDEN:
logger.warning("qBittorrent authentication failed (HTTP 403)")
@@ -257,6 +296,41 @@ class QBittorrentClient(DownloadClient):
response.raise_for_status()
torrents = response.json()
return [SimpleNamespace(**t) for t in torrents], None
try:
primary_params: dict[str, str] = {}
if torrent_hash:
primary_params["hashes"] = torrent_hash
response = do_request(primary_params)
torrents, error = parse_response(response, request_params=primary_params)
if error:
return [], error
if torrent_hash and not torrents:
# Fallback 1: list by configured category
category_params: dict[str, str] = {}
if self._category:
category_params["category"] = self._category
category_response = do_request(category_params)
category_torrents, category_error = parse_response(
category_response, request_params=category_params
)
if category_error:
return [], category_error
if category_torrents:
return category_torrents, None
# Fallback 2: list everything (handles per-task categories like audiobooks)
all_response = do_request({})
all_torrents, all_error = parse_response(all_response, request_params={})
if all_error:
return [], all_error
return all_torrents, None
except requests.exceptions.HTTPError as e:
status = getattr(getattr(e, "response", None), "status_code", None)
if status:
@@ -281,100 +355,8 @@ class QBittorrentClient(DownloadClient):
except _QBITTORRENT_CLIENT_ERRORS as e:
logger.debug("Failed to get torrents info: %s", e)
return [], f"qBittorrent API error: {type(e).__name__}: {e}"
def _get_torrent_info(self, download_id: str) -> tuple[SimpleNamespace | None, str | None]:
"""Get one torrent by its current qBittorrent hash."""
torrents, error = self._request_torrent_info_records({"hashes": download_id})
if error or not torrents:
return None, error
return (
next(
(
torrent
for torrent in torrents
if isinstance(getattr(torrent, "hash", None), str)
and _hashes_match(torrent.hash, download_id)
),
None,
),
None,
)
def _list_torrents_by_category(
self, category: str | None
) -> tuple[list[SimpleNamespace], str | None]:
"""List torrent records in a category, or all records when unset."""
params = {"category": category} if category else {}
return self._request_torrent_info_records(params)
def _resolve_torrent(
self, download_id: str, category: str | None = None
) -> tuple[SimpleNamespace | None, str | None]:
"""Resolve any known torrent identity to its current qBittorrent record."""
torrent, error = self._get_torrent_info(download_id)
if error or torrent:
return torrent, error
categories = [candidate for candidate in (category, self._category) if candidate]
for candidate in dict.fromkeys(categories):
torrents, error = self._list_torrents_by_category(candidate)
if error:
return None, error
torrent = next(
(item for item in torrents if _torrent_matches_download_id(item, download_id)),
None,
)
if torrent:
return torrent, None
torrents, error = self._list_torrents_by_category(None)
if error:
return None, error
return (
next(
(item for item in torrents if _torrent_matches_download_id(item, download_id)),
None,
),
None,
)
def _list_category_hashes(self, category: str | None) -> set[str] | None:
"""Snapshot the hashes qBittorrent currently reports for a category."""
torrents, error = self._list_torrents_by_category(category)
if error:
logger.debug("Could not snapshot qBittorrent torrents: %s", error)
return None
return {str(torrent.hash).lower() for torrent in torrents if getattr(torrent, "hash", None)}
def _discover_added_torrent_hash(
self,
name: str,
category: str | None,
known_hashes: set[str] | None,
) -> str | None:
"""Recover the hash of a torrent that was added without a known info_hash.
A `known_hashes` of None means the pre-add snapshot failed, so only a
torrent matching the requested rename can identify the new arrival.
"""
for _ in range(20):
torrents, error = self._list_torrents_by_category(category)
if error:
logger.debug("qBittorrent hash discovery: %s", error)
else:
new_torrents = [
torrent
for torrent in torrents
if getattr(torrent, "hash", None)
and (known_hashes is None or str(torrent.hash).lower() not in known_hashes)
]
for torrent in new_torrents:
if getattr(torrent, "name", None) == name:
return str(torrent.hash).lower()
if known_hashes is not None and len(new_torrents) == 1:
return str(new_torrents[0].hash).lower()
time.sleep(0.5)
return None
else:
return torrents, None
@staticmethod
def is_configured() -> bool:
@@ -386,7 +368,7 @@ class QBittorrentClient(DownloadClient):
def test_connection(self) -> tuple[bool, str]:
"""Test connection to qBittorrent."""
try:
self._ensure_authenticated()
self._client.auth_log_in()
api_version = self._client.app.web_api_version
except _QBITTORRENT_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
@@ -443,10 +425,6 @@ class QBittorrentClient(DownloadClient):
expected_hash = torrent_info.info_hash
torrent_data = torrent_info.torrent_data
known_hashes: set[str] | None = None
if not expected_hash:
known_hashes = self._list_category_hashes(category)
# Per-torrent seeding limits from indexer
seeding_time_limit_value = kwargs.get("seeding_time_limit")
seeding_time_limit = coerce_optional_int(seeding_time_limit_value)
@@ -481,36 +459,27 @@ class QBittorrentClient(DownloadClient):
result_text = _normalize_add_result(result)
logger.debug("qBittorrent add result: %s", result_text)
if not expected_hash:
_raise_runtime_error("Could not determine torrent hash from URL")
if _is_explicit_add_failure(result):
_raise_runtime_error(f"Failed to add torrent: {result_text}")
if not expected_hash:
# qBittorrent fetches .torrent URLs itself, so the add can succeed
# even when no hash could be extracted up front. Recover it by
# watching for the new torrent to appear.
expected_hash = self._discover_added_torrent_hash(name, category, known_hashes)
if not expected_hash:
message = "Could not determine torrent hash from URL"
if torrent_info.fetch_error:
message = f"{message} (torrent file fetch failed: {torrent_info.fetch_error})"
_raise_runtime_error(message)
# Wait until qBittorrent has resolved magnet metadata so the returned
# hash is its stable primary torrent ID, which may differ from the v1 hash.
for _ in range(20):
torrent, error = self._resolve_torrent(expected_hash, category)
# Some qBittorrent-compatible clients return HTTP 200 with an empty body
# instead of qBittorrent's literal "Ok." response. Prefer verifying that
# the torrent becomes visible over trusting the response body alone.
for _ in range(10):
loaded, error = self._is_torrent_loaded(expected_hash)
if error:
logger.debug("qBittorrent add_download: %s", error)
elif torrent and getattr(torrent, "state", None) not in _METADATA_DOWNLOAD_STATES:
torrent_hash = getattr(torrent, "hash", None)
if isinstance(torrent_hash, str) and torrent_hash:
logger.info("Added torrent: %s", torrent_hash)
return torrent_hash.lower()
if loaded:
logger.info("Added torrent: %s", expected_hash)
return expected_hash.lower()
time.sleep(0.5)
_raise_runtime_error(
"Torrent metadata resolution was not confirmed within the visibility grace period "
f"(response={result_text})"
logger.warning(
"Torrent add was not confirmed within the visibility grace period (response=%s), returning expected hash",
result_text,
)
except _QBITTORRENT_CLIENT_ERRORS:
logger.exception("qBittorrent add failed")
@@ -529,9 +498,19 @@ class QBittorrentClient(DownloadClient):
"""
try:
torrent, error = self._get_torrent_info(download_id)
torrents, error = self._get_torrents_info(download_id)
if error:
return DownloadStatus.error(error)
torrent = next(
(
t
for t in torrents
if isinstance(getattr(t, "hash", None), str)
and _hashes_match(t.hash, download_id)
),
None,
)
if not torrent:
return DownloadStatus.error("Torrent not found in qBittorrent")
@@ -625,26 +604,6 @@ class QBittorrentClient(DownloadClient):
else:
return True
def set_category(self, download_id: str, category: str) -> bool:
"""Assign a category to a torrent in qBittorrent."""
try:
try:
self._client.torrents_create_category(name=category)
except _QBITTORRENT_CLIENT_ERRORS as e:
if "Conflict" not in type(e).__name__ and "409" not in str(e):
logger.debug("Could not create category '%s': %s", category, e)
self._client.torrents_set_category(
torrent_hashes=download_id,
category=category,
)
logger.info("Set qBittorrent category for %s to '%s'", download_id, category)
except _QBITTORRENT_CLIENT_ERRORS as e:
self._log_error("set_category", e)
return False
else:
return True
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where torrent files are located.
@@ -657,10 +616,20 @@ class QBittorrentClient(DownloadClient):
- join `save_path` with the torrent's top-level directory
"""
try:
torrent, error = self._get_torrent_info(download_id)
torrents, error = self._get_torrents_info(download_id)
if error:
logger.debug("qBittorrent get_download_path: %s", error)
return None
torrent = next(
(
t
for t in torrents
if isinstance(getattr(t, "hash", None), str)
and _hashes_match(t.hash, download_id)
),
None,
)
if not torrent:
return None
@@ -706,11 +675,11 @@ class QBittorrentClient(DownloadClient):
import os
def get_with_auth(url: str, params: dict[str, str]) -> requests.Response:
self._ensure_authenticated()
self._client.auth_log_in()
resp = self._client._session.get(url, params=params, timeout=10)
if resp.status_code == _HTTP_STATUS_FORBIDDEN and self._can_reauthenticate:
if resp.status_code == _HTTP_STATUS_FORBIDDEN:
logger.debug("qBittorrent returned 403; re-authenticating and retrying")
self._ensure_authenticated()
self._client.auth_log_in()
resp = self._client._session.get(url, params=params, timeout=10)
return resp
@@ -767,19 +736,23 @@ class QBittorrentClient(DownloadClient):
if not torrent_info.info_hash:
return None
for _ in range(20):
torrent, error = self._resolve_torrent(torrent_info.info_hash, category)
if error:
logger.debug("qBittorrent find_existing: %s", error)
return None
if not torrent:
return None
if getattr(torrent, "state", None) not in _METADATA_DOWNLOAD_STATES:
torrent_hash = getattr(torrent, "hash", None)
if isinstance(torrent_hash, str) and torrent_hash:
torrent_hash = torrent_hash.lower()
return (torrent_hash, self.get_status(torrent_hash))
time.sleep(0.5)
torrents, error = self._get_torrents_info(torrent_info.info_hash)
if error:
logger.debug("qBittorrent find_existing: %s", error)
return None
torrent = next(
(
t
for t in torrents
if isinstance(getattr(t, "hash", None), str)
and _hashes_match(t.hash, torrent_info.info_hash)
),
None,
)
if torrent and isinstance(getattr(torrent, "hash", None), str):
torrent_hash = torrent.hash
return (torrent_hash.lower(), self.get_status(torrent_hash.lower()))
except _QBITTORRENT_CLIENT_ERRORS as e:
logger.debug("Error checking for existing torrent: %s", e)
return None
-517
View File
@@ -1,517 +0,0 @@
"""Real-Debrid debrid service client for Shelfmark.
Routes magnet links through the Real-Debrid REST API (v1.0) to download
torrent content via Real-Debrid's CDN infrastructure.
"""
from __future__ import annotations
import shutil
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, ClassVar, NoReturn
import requests
from shelfmark.config.env import TMP_DIR
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.download.clients import (
DownloadClient,
DownloadState,
DownloadStatus,
register_client,
)
from shelfmark.download.clients._coercion import config_text
from shelfmark.download.http import download_url
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_API_BASE = "https://api.real-debrid.com/rest/1.0"
_REALDEBRID_CLIENT_ERRORS = (
AttributeError,
OSError,
requests.exceptions.RequestException,
RuntimeError,
TypeError,
ValueError,
)
# Real-Debrid torrent status values.
_STATUS_DOWNLOADING = frozenset(
{
"magnet_conversion",
"waiting_files_selection",
"downloading",
"compressing",
"uploading",
}
)
_STATUS_READY = "downloaded"
_STATUS_ERROR = frozenset({"error", "virus", "dead"})
# Timeouts for API calls.
_API_TIMEOUT = 30
_STATUS_TIMEOUT = 15
# File extensions recognised as book or audiobook content.
_BOOK_EXTENSIONS = (
".aac",
".azw",
".azw3",
".cbr",
".cbz",
".djvu",
".doc",
".docx",
".epub",
".fb2",
".flac",
".lit",
".m4a",
".m4b",
".mobi",
".mp3",
".ogg",
".opus",
".pdf",
".rtf",
".txt",
".wma",
)
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
@dataclass
class _DownloadState:
"""Internal mutable state for an in-progress Real-Debrid download."""
torrent_id: str
name: str
target_dir: Path
phase: str = "uploading"
error_message: str | None = None
progress: float = 0.0
download_thread: threading.Thread | None = None
lock: threading.Lock = field(default_factory=threading.Lock)
@register_client("torrent")
class RealDebridClient(DownloadClient):
"""Real-Debrid debrid service client.
Downloads torrent content by uploading magnet links to Real-Debrid,
selecting all files for download on their servers, then unrestricting
and fetching the resulting files via direct HTTP download from
Real-Debrid's CDN.
API documentation: https://api.real-debrid.com/
"""
protocol = "torrent"
name = "realdebrid"
_downloads: ClassVar[dict[str, _DownloadState]] = {}
_downloads_lock = threading.Lock()
def __init__(self) -> None:
self._api_key = config_text(config.get("REALDEBRID_API_KEY", ""))
def _auth_headers(self) -> dict[str, str]:
"""Return Authorization header dict for API requests."""
return {"Authorization": f"Bearer {self._api_key}"}
# ------------------------------------------------------------------
# DownloadClient interface
# ------------------------------------------------------------------
@staticmethod
def is_configured() -> bool:
"""Return True when Real-Debrid is selected and an API key exists."""
client = config_text(config.get("PROWLARR_TORRENT_CLIENT", ""))
api_key = config_text(config.get("REALDEBRID_API_KEY", ""))
return client == "realdebrid" and bool(api_key)
def test_connection(self) -> tuple[bool, str]:
"""Validate the API key and check Premium subscription status."""
if not self._api_key:
return False, "Real-Debrid API Key is required"
try:
url = f"{_API_BASE}/user"
resp = requests.get(
url,
headers=self._auth_headers(),
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(url),
)
resp.raise_for_status()
user = resp.json()
username = user.get("username", "Unknown")
account_type = user.get("type", "free")
if account_type != "premium":
return (
False,
f"Real-Debrid user '{username}' does not have "
f"a Premium subscription (type: {account_type})",
)
except _REALDEBRID_CLIENT_ERRORS as e:
return False, f"Connection failed: {e}"
else:
return True, f"Connected to Real-Debrid as '{username}' (Premium)"
def add_download(
self,
url: str,
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs: object,
) -> str:
"""Upload a magnet link to Real-Debrid and select all files."""
if not self._api_key:
msg = "Real-Debrid API key is not configured"
raise RuntimeError(msg)
magnet_link = url
if not magnet_link.startswith("magnet:") and expected_hash:
magnet_link = f"magnet:?xt=urn:btih:{expected_hash}"
add_url = f"{_API_BASE}/torrents/addMagnet"
try:
resp = requests.post(
add_url,
headers=self._auth_headers(),
data={"magnet": magnet_link},
timeout=_API_TIMEOUT,
verify=get_ssl_verify(add_url),
)
resp.raise_for_status()
data = resp.json()
torrent_id = str(data.get("id", ""))
if not torrent_id:
msg = "No torrent ID returned from Real-Debrid"
_raise_runtime_error(msg)
# Select all files so Real-Debrid starts downloading the torrent
select_url = f"{_API_BASE}/torrents/selectFiles/{torrent_id}"
sel_resp = requests.post(
select_url,
headers=self._auth_headers(),
data={"files": "all"},
timeout=_API_TIMEOUT,
verify=get_ssl_verify(select_url),
)
sel_resp.raise_for_status()
target_dir = TMP_DIR / f"realdebrid_{torrent_id}"
target_dir.mkdir(parents=True, exist_ok=True)
state = _DownloadState(
torrent_id=torrent_id,
name=name,
target_dir=target_dir,
phase="waiting_rd",
)
with self._downloads_lock:
self._downloads[torrent_id] = state
logger.info(
"Added torrent to Real-Debrid: ID %s (%s)",
torrent_id,
name,
)
except Exception:
logger.exception("Failed to upload magnet to Real-Debrid")
raise
else:
return torrent_id
def get_status(self, download_id: str) -> DownloadStatus:
"""Poll Real-Debrid for torrent status and drive the download."""
state = self._ensure_state(download_id)
# Return cached terminal / in-flight states immediately.
with state.lock:
if state.phase == "error":
return DownloadStatus.error(
state.error_message or "Real-Debrid error",
)
if state.phase == "complete":
return DownloadStatus(
progress=100.0,
state=DownloadState.COMPLETE,
message="Complete",
complete=True,
file_path=str(state.target_dir),
)
if state.phase == "downloading_http":
return DownloadStatus(
progress=state.progress,
state=DownloadState.DOWNLOADING,
message="Downloading files via HTTP...",
complete=False,
file_path=None,
)
# Query Real-Debrid for torrent info.
try:
info_url = f"{_API_BASE}/torrents/info/{download_id}"
resp = requests.get(
info_url,
headers=self._auth_headers(),
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(info_url),
)
resp.raise_for_status()
info = resp.json()
return self._handle_torrent_info(info, state)
except Exception as e:
logger.exception(
"Error checking Real-Debrid status for %s",
download_id,
)
return DownloadStatus.error(str(e))
def remove(
self,
download_id: str,
*,
delete_files: bool = False,
) -> bool:
"""Delete the torrent from Real-Debrid and clean up local files."""
try:
url = f"{_API_BASE}/torrents/delete/{download_id}"
requests.delete(
url,
headers=self._auth_headers(),
timeout=_STATUS_TIMEOUT,
verify=get_ssl_verify(url),
)
except _REALDEBRID_CLIENT_ERRORS as e:
logger.warning("Failed to delete torrent from Real-Debrid: %s", e)
with self._downloads_lock:
state = self._downloads.pop(download_id, None)
if state and state.target_dir.exists():
shutil.rmtree(state.target_dir, ignore_errors=True)
return True
def get_download_path(self, download_id: str) -> str | None:
"""Return the local directory containing downloaded files."""
with self._downloads_lock:
state = self._downloads.get(download_id)
if state and state.phase == "complete":
return str(state.target_dir)
target_dir = TMP_DIR / f"realdebrid_{download_id}"
if target_dir.exists():
return str(target_dir)
return None
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _ensure_state(self, download_id: str) -> _DownloadState:
"""Get or create download state for the given torrent ID."""
with self._downloads_lock:
state = self._downloads.get(download_id)
if state:
return state
target_dir = TMP_DIR / f"realdebrid_{download_id}"
state = _DownloadState(
torrent_id=download_id,
name=f"Download {download_id}",
target_dir=target_dir,
phase="waiting_rd",
)
with self._downloads_lock:
self._downloads[download_id] = state
return state
def _handle_torrent_info(
self,
info: dict[str, Any],
state: _DownloadState,
) -> DownloadStatus:
"""Map Real-Debrid torrent info to a DownloadStatus."""
status = info.get("status", "")
if status in _STATUS_DOWNLOADING:
progress = float(info.get("progress", 0.0))
speed = int(info.get("speed", 0))
filename = info.get("filename", state.name)
return DownloadStatus(
progress=progress * 0.5,
state=DownloadState.DOWNLOADING,
message=f"Real-Debrid downloading torrent ({filename})",
complete=False,
file_path=None,
download_speed=speed,
)
if status == _STATUS_READY:
links = info.get("links", [])
files = info.get("files", [])
self._maybe_start_download_thread(state, links, files)
return DownloadStatus(
progress=50.0,
state=DownloadState.DOWNLOADING,
message="Real-Debrid ready, retrieving files...",
complete=False,
file_path=None,
)
# Terminal error from Real-Debrid.
error_txt = f"Real-Debrid status error: {status}"
with state.lock:
state.phase = "error"
state.error_message = error_txt
return DownloadStatus.error(error_txt)
def _maybe_start_download_thread(
self,
state: _DownloadState,
links: list[str],
files: list[dict[str, Any]],
) -> None:
"""Spawn a background thread to unrestrict and download files."""
with state.lock:
already_running = state.phase in (
"unrestricting",
"downloading_http",
"complete",
)
thread_alive = state.download_thread is not None and state.download_thread.is_alive()
if already_running or thread_alive:
return
state.phase = "unrestricting"
t = threading.Thread(
target=self._process_and_download,
args=(state, links, files),
daemon=True,
)
state.download_thread = t
t.start()
# ------------------------------------------------------------------
# File download pipeline
# ------------------------------------------------------------------
def _process_and_download(
self,
state: _DownloadState,
links: list[str],
files: list[dict[str, Any]],
) -> None:
"""Unrestrict links and download files via HTTP.
Runs in a background thread spawned by ``_maybe_start_download_thread``.
"""
try:
if not links:
msg = "No download links returned by Real-Debrid"
_raise_runtime_error(msg)
# Match selected files with links
selected_files = [f for f in files if f.get("selected") == 1]
# Filter relevant ebook / audiobook files
relevant_indices: list[int] = []
for i, f_info in enumerate(selected_files):
path_str = f_info.get("path", "").lower()
if path_str.endswith(_BOOK_EXTENSIONS):
relevant_indices.append(i)
if not relevant_indices:
relevant_indices = list(range(len(links)))
with state.lock:
state.phase = "downloading_http"
total = len(relevant_indices)
for idx, rel_idx in enumerate(relevant_indices):
if rel_idx >= len(links):
continue
link = links[rel_idx]
# Unrestrict the Real-Debrid link to get direct CDN download URL
unrestrict_url = f"{_API_BASE}/unrestrict/link"
unl_resp = requests.post(
unrestrict_url,
headers=self._auth_headers(),
data={"link": link},
timeout=_API_TIMEOUT,
verify=get_ssl_verify(unrestrict_url),
)
unl_resp.raise_for_status()
unl_data = unl_resp.json()
direct_url = unl_data.get("download")
filename = unl_data.get("filename")
if not direct_url:
msg = f"Failed to unrestrict Real-Debrid link: {link}"
_raise_runtime_error(msg)
# Determine relative file path
if rel_idx < len(selected_files):
rel_path_str = selected_files[rel_idx].get("path", "").lstrip("/")
rel_path = Path(rel_path_str)
else:
rel_path = Path(filename or f"file_{idx + 1}")
dest = state.target_dir / rel_path
dest.parent.mkdir(parents=True, exist_ok=True)
logger.info(
"Downloading Real-Debrid file %d/%d: %s",
idx + 1,
total,
rel_path,
)
buf = download_url(
direct_url,
referer="https://real-debrid.com/",
)
if not buf:
msg = f"Failed to download from {direct_url}"
_raise_runtime_error(msg)
with dest.open("wb") as fh:
fh.write(buf.getvalue())
with state.lock:
state.progress = 50.0 + (idx + 1) / total * 50.0
with state.lock:
state.phase = "complete"
state.progress = 100.0
logger.info(
"Real-Debrid download complete for ID %s at %s",
state.torrent_id,
state.target_dir,
)
except Exception:
logger.exception(
"Error in Real-Debrid download for ID %s",
state.torrent_id,
)
with state.lock:
state.phase = "error"
state.error_message = str(
state.error_message or "Download failed",
)
+6 -95
View File
@@ -4,7 +4,6 @@ Uses xmlrpc to communicate with rTorrent's RPC interface.
"""
import ssl
import time
import xmlrpc.client as stdlib_xmlrpc_client
from typing import Any, NoReturn, Protocol, cast
from urllib.parse import urlparse
@@ -47,13 +46,7 @@ class _RTorrentLoadProtocol(Protocol):
def start(self, target: str, url: str, commands: str) -> object: ...
class _RTorrentCustom1Protocol(Protocol):
def set(self, download_id: str, value: str) -> object: ...
class _RTorrentDownloadProtocol(Protocol):
custom1: _RTorrentCustom1Protocol
def multicall2(self, *args: object) -> list[list[Any]]: ...
def delete_tied(self, download_id: str) -> object: ...
@@ -122,7 +115,6 @@ class RTorrentClient(DownloadClient):
self._rpc = _create_rtorrent_server_proxy(self._base_url)
self._download_dir = config_text(config.get("RTORRENT_DOWNLOAD_DIR", ""))
self._label = config_text(config.get("RTORRENT_LABEL", ""))
self._audiobook_label = config_text(config.get("RTORRENT_AUDIOBOOK_LABEL", ""))
@staticmethod
def is_configured() -> bool:
@@ -167,17 +159,9 @@ class RTorrentClient(DownloadClient):
try:
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
known_hashes: set[str] | None = None
if not (torrent_info.info_hash or expected_hash):
known_hashes = self._list_torrent_hashes()
commands = []
is_audiobook = kwargs.get("content_type") == "audiobook"
default_label = (
self._audiobook_label if is_audiobook and self._audiobook_label else self._label
)
label = category or default_label
label = category or self._label
if label:
logger.debug("Setting rTorrent label: %s", label)
commands.append(f"d.custom1.set={label}")
@@ -207,15 +191,7 @@ class RTorrentClient(DownloadClient):
torrent_hash = torrent_info.info_hash or expected_hash
if not torrent_hash:
# rTorrent fetches .torrent URLs itself, so the add can succeed
# even when no hash could be extracted up front. Recover it by
# watching for the new download to appear.
torrent_hash = self._discover_added_torrent_hash(name, label, known_hashes)
if not torrent_hash:
message = "Could not determine torrent hash from URL"
if torrent_info.fetch_error:
message = f"{message} (torrent file fetch failed: {torrent_info.fetch_error})"
_raise_runtime_error(message)
_raise_runtime_error("Could not determine torrent hash from URL")
logger.debug("Added torrent to rTorrent: %s", torrent_hash)
@@ -338,14 +314,12 @@ class RTorrentClient(DownloadClient):
"""
try:
# rtorrent is somehow case sensitive and requires uppercase hashes for look
torrent_hash = download_id.upper()
if delete_files:
self._rpc.d.delete_tied(torrent_hash)
self._rpc.d.erase(torrent_hash)
self._rpc.d.delete_tied(download_id)
self._rpc.d.erase(download_id)
else:
self._rpc.d.stop(torrent_hash)
self._rpc.d.erase(torrent_hash)
self._rpc.d.stop(download_id)
self._rpc.d.erase(download_id)
logger.info(
"Removed torrent from rTorrent: %s%s",
@@ -359,19 +333,6 @@ class RTorrentClient(DownloadClient):
else:
return True
def set_category(self, download_id: str, category: str) -> bool:
"""Assign a label to a torrent using rTorrent's custom1 field."""
try:
# rtorrent is somehow case sensitive and requires uppercase hashes for look
self._rpc.d.custom1.set(download_id.upper(), category)
logger.info("Set rTorrent label for %s to '%s'", download_id, category)
except _RTORRENT_CLIENT_ERRORS as e:
error_type = type(e).__name__
logger.exception("rTorrent set_category failed (%s)", error_type)
return False
else:
return True
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where torrent files are located.
@@ -421,56 +382,6 @@ class RTorrentClient(DownloadClient):
except _RTORRENT_CLIENT_ERRORS:
return "/downloads"
def _list_torrent_hashes(self) -> set[str] | None:
"""Snapshot the hashes rTorrent currently reports."""
try:
all_torrents = self._rpc.d.multicall2("", "", "d.hash=")
except _RTORRENT_CLIENT_ERRORS as e:
logger.debug("Could not snapshot rTorrent downloads: %s", e)
return None
return {str(row[0]).lower() for row in all_torrents if row and row[0]}
def _discover_added_torrent_hash(
self,
name: str,
label: str,
known_hashes: set[str] | None,
) -> str | None:
"""Recover the hash of a torrent that was added without a known info_hash.
rTorrent fetches .torrent URLs itself, so the add can succeed even when
no hash could be extracted up front. A `known_hashes` of None means the
pre-add snapshot failed, so only an exact name match can identify the
new arrival.
"""
for _ in range(20):
try:
all_torrents = self._rpc.d.multicall2("", "", "d.hash=", "d.name=", "d.custom1=")
except _RTORRENT_CLIENT_ERRORS as e:
logger.debug("rTorrent hash discovery: %s", e)
else:
new_torrents = [
row
for row in all_torrents
if row
and row[0]
and (known_hashes is None or str(row[0]).lower() not in known_hashes)
]
# The label set at add time distinguishes concurrent arrivals,
# but rTorrent may not have applied it yet, so it only ever
# narrows a non-empty candidate list.
if label:
labeled = [row for row in new_torrents if len(row) > 2 and row[2] == label]
if labeled:
new_torrents = labeled
for row in new_torrents:
if len(row) > 1 and row[1] == name:
return str(row[0]).lower()
if known_hashes is not None and len(new_torrents) == 1:
return str(new_torrents[0][0]).lower()
time.sleep(0.5)
return None
def _get_torrent_path(self, download_id: str) -> str | None:
"""Get the file path of a torrent by hash.
+3 -102
View File
@@ -159,7 +159,6 @@ def _test_qbittorrent_connection(current_values: dict[str, Any] | None = None) -
raw_url = _resolve_string_setting(current_values, config.get, "QBITTORRENT_URL")
username = _resolve_string_setting(current_values, config.get, "QBITTORRENT_USERNAME")
password = _resolve_string_setting(current_values, config.get, "QBITTORRENT_PASSWORD")
api_key = _resolve_string_setting(current_values, config.get, "QBITTORRENT_API_KEY")
if not raw_url:
return {"success": False, "message": "qBittorrent URL is required"}
@@ -175,7 +174,6 @@ def _test_qbittorrent_connection(current_values: dict[str, Any] | None = None) -
host=url,
username=username,
password=password,
api_key=api_key or None,
VERIFY_WEBUI_CERTIFICATE=get_ssl_verify(url),
)
client.auth_log_in()
@@ -183,18 +181,9 @@ def _test_qbittorrent_connection(current_values: dict[str, Any] | None = None) -
except ImportError:
return {"success": False, "message": "qbittorrent-api package not installed"}
except _QBITTORRENT_SETTINGS_ERRORS as e:
if isinstance(e, _QBittorrentLoginFailed):
# LoginFailed carries no message of its own, so name the rejected credential.
rejected = "API key" if api_key else "username or password"
return {"success": False, "message": f"qBittorrent rejected the {rejected}"}
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
# Both credentials can be set at once, so name the one that actually authenticated.
used = " using the API key" if api_key else ""
return {
"success": True,
"message": f"Connected to qBittorrent (API v{api_version}){used}",
}
return {"success": True, "message": f"Connected to qBittorrent (API v{api_version})"}
def _test_transmission_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
@@ -531,40 +520,6 @@ def _test_sabnzbd_connection(current_values: dict[str, Any] | None = None) -> di
return {"success": True, "message": f"Connected to SABnzbd {version}"}
def _test_alldebrid_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the AllDebrid API connection using current form values."""
from shelfmark.core.config import config
from shelfmark.download.clients.alldebrid import AllDebridClient
current_values = current_values or {}
api_key = _resolve_string_setting(current_values, config.get, "ALLDEBRID_API_KEY")
if not api_key:
return {"success": False, "message": "AllDebrid API Key is required"}
client = AllDebridClient()
client._api_key = api_key
success, message = client.test_connection()
return {"success": success, "message": message}
def _test_realdebrid_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the Real-Debrid API connection using current form values."""
from shelfmark.core.config import config
from shelfmark.download.clients.realdebrid import RealDebridClient
current_values = current_values or {}
api_key = _resolve_string_setting(current_values, config.get, "REALDEBRID_API_KEY")
if not api_key:
return {"success": False, "message": "Real-Debrid API Key is required"}
client = RealDebridClient()
client._api_key = api_key
success, message = client.test_connection()
return {"success": success, "message": message}
# ==================== Download Clients Tab ====================
@@ -589,45 +544,13 @@ def prowlarr_clients_settings() -> list[SettingsField]:
description="Choose which torrent client to use",
options=[
{"value": "", "label": "None"},
{"value": "alldebrid", "label": "AllDebrid"},
{"value": "qbittorrent", "label": "qBittorrent"},
{"value": "realdebrid", "label": "Real-Debrid"},
{"value": "transmission", "label": "Transmission"},
{"value": "deluge", "label": "Deluge"},
{"value": "rtorrent", "label": "rTorrent"},
],
default="",
),
# --- AllDebrid Settings ---
PasswordField(
key="ALLDEBRID_API_KEY",
label="API Key",
description="AllDebrid API Key (apiv4) from your AllDebrid account settings",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "alldebrid"},
),
ActionButton(
key="test_alldebrid",
label="Test Connection",
description="Verify your AllDebrid configuration",
style="primary",
callback=_test_alldebrid_connection,
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "alldebrid"},
),
# --- Real-Debrid Settings ---
PasswordField(
key="REALDEBRID_API_KEY",
label="API Key",
description="Real-Debrid API Key (Secret Token) from your Real-Debrid account settings",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "realdebrid"},
),
ActionButton(
key="test_realdebrid",
label="Test Connection",
description="Verify your Real-Debrid configuration",
style="primary",
callback=_test_realdebrid_connection,
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "realdebrid"},
),
# --- qBittorrent Settings ---
TextField(
key="QBITTORRENT_URL",
@@ -649,12 +572,6 @@ def prowlarr_clients_settings() -> list[SettingsField]:
description="qBittorrent Web UI password",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "qbittorrent"},
),
PasswordField(
key="QBITTORRENT_API_KEY",
label="API Key",
description="Found in qBittorrent: Options > Web UI > API Key (qBittorrent 5.2.0+). Used instead of the username and password when set.",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "qbittorrent"},
),
ActionButton(
key="test_qbittorrent",
label="Test Connection",
@@ -831,18 +748,11 @@ def prowlarr_clients_settings() -> list[SettingsField]:
TextField(
key="RTORRENT_LABEL",
label="Book Label",
description="Label to assign to ebook downloads in rTorrent",
description="Label to assign to book downloads in rTorrent",
placeholder="cwabd",
default="cwabd",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
),
TextField(
key="RTORRENT_AUDIOBOOK_LABEL",
label="Audiobook Label",
description="Label to assign to audiobook downloads in rTorrent (falls back to Book Label if not set)",
placeholder="audiobooks",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
),
TextField(
key="RTORRENT_DOWNLOAD_DIR",
label="Download Directory",
@@ -854,23 +764,14 @@ def prowlarr_clients_settings() -> list[SettingsField]:
SelectField(
key="PROWLARR_TORRENT_ACTION",
label="Torrent Completion Action",
description="Choose whether to keep, remove, or move the torrent to another category or label after import",
description="Remove deletes the torrent from your client immediately after import (stops seeding, files are kept); Keep leaves it in the client to continue seeding",
options=[
{"value": "keep", "label": "Keep"},
{"value": "remove", "label": "Remove"},
{"value": "change_category", "label": "Change Category"},
],
default="keep",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "notEmpty": True},
),
TextField(
key="PROWLARR_TORRENT_POST_IMPORT_CATEGORY",
label="Post-Import Category",
description="Category or label to assign after a successful import",
placeholder="imported",
default="",
show_when={"field": "PROWLARR_TORRENT_ACTION", "value": "change_category"},
),
# --- Usenet Client Selection ---
HeadingField(
key="usenet_heading",
+38 -102
View File
@@ -5,10 +5,8 @@ from __future__ import annotations
import base64
import hashlib
import re
import time
from binascii import Error as BinasciiError
from dataclasses import dataclass
from threading import Lock
from urllib.parse import ParseResult, parse_qs, urljoin, urlparse
import requests
@@ -21,7 +19,6 @@ from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_MAGNET_RESPONSE_MAX_BYTES = 2000
_TORRENT_FETCH_MAX_REDIRECTS = 5
_BASE32_BTMH_TAG_BYTES = 34
_BTIH_INFO_BYTE_HEX = 0x20
_BTIH_PREFIX_BYTE = 0x12
@@ -38,15 +35,6 @@ _TORRENT_FETCH_ERRORS = (
_TORRENT_PARSE_ERRORS = (IndexError, KeyError, TypeError, ValueError)
_TRUSTED_TORRENT_FETCH_URL_CONFIG_KEYS = ("PROWLARR_URL", "NEWZNAB_URL")
# Successful torrent fetches are reused for a short window so one add attempt
# hits the download link only once. Tracker download links (e.g. private
# trackers behind Prowlarr's proxy) can be slow, rate-limited, or single-use,
# and both find_existing() and add_download() resolve the same URL (#1111).
_TORRENT_FETCH_CACHE_TTL_SECONDS = 120.0
_TORRENT_FETCH_CACHE_MAX_ENTRIES = 8
_torrent_fetch_cache_lock = Lock()
_torrent_fetch_cache: dict[str, tuple[float, TorrentInfo]] = {}
type BencodeValue = dict[str | bytes, BencodeValue] | list[BencodeValue] | int | bytes | str
@@ -66,9 +54,6 @@ class TorrentInfo:
magnet_url: str | None = None
"""The actual magnet URL, if available."""
fetch_error: str | None = None
"""Why fetching the .torrent URL failed, or None if it succeeded/was skipped."""
def with_info_hash(self, info_hash: str | None) -> TorrentInfo:
"""Return a copy with the info_hash replaced when provided."""
if info_hash:
@@ -77,7 +62,6 @@ class TorrentInfo:
torrent_data=self.torrent_data,
is_magnet=self.is_magnet,
magnet_url=self.magnet_url,
fetch_error=self.fetch_error,
)
return self
@@ -111,58 +95,10 @@ def extract_torrent_info(
# Not a magnet - try to fetch and parse the .torrent file
if not fetch_torrent:
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
if not _is_trusted_torrent_fetch_url(url):
logger.debug("Skipping torrent prefetch for untrusted URL: %s...", url[:80])
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
info = _get_cached_torrent_fetch(url)
if info is None:
info = _fetch_torrent_info(url)
if info.fetch_error is None:
_store_cached_torrent_fetch(url, info)
return info.with_info_hash(info.info_hash or expected_hash)
def _get_cached_torrent_fetch(url: str) -> TorrentInfo | None:
with _torrent_fetch_cache_lock:
entry = _torrent_fetch_cache.get(url)
if entry is None:
return None
fetched_at, info = entry
if time.monotonic() - fetched_at > _TORRENT_FETCH_CACHE_TTL_SECONDS:
del _torrent_fetch_cache[url]
return None
logger.debug("Reusing recently fetched torrent data for: %s...", url[:80])
return info
def _store_cached_torrent_fetch(url: str, info: TorrentInfo) -> None:
with _torrent_fetch_cache_lock:
_torrent_fetch_cache[url] = (time.monotonic(), info)
while len(_torrent_fetch_cache) > _TORRENT_FETCH_CACHE_MAX_ENTRIES:
oldest_url = min(_torrent_fetch_cache, key=lambda key: _torrent_fetch_cache[key][0])
del _torrent_fetch_cache[oldest_url]
def clear_torrent_fetch_cache() -> None:
"""Drop all cached torrent fetches (used by tests)."""
with _torrent_fetch_cache_lock:
_torrent_fetch_cache.clear()
def _fetch_torrent_info(url: str) -> TorrentInfo:
"""Fetch a .torrent URL and parse out the info_hash and raw torrent data.
On failure, the returned TorrentInfo carries the reason in `fetch_error`
so callers can surface it instead of a generic hash error.
"""
# A release source can legitimately hand us a download URL on a different
# origin than the configured Prowlarr/Newznab endpoint (e.g. a direct
# tracker link, or Prowlarr reached through a separate proxy), and a trusted
# Prowlarr download URL commonly redirects to the indexer's own download
# link. We still need to fetch the .torrent to recover the info_hash when
# the source did not provide one, so the prefetch runs regardless of origin
# and follows cross-origin redirects. The Prowlarr API key, however, is
# re-evaluated per hop and only ever sent to a trusted origin so it can
# never leak to an arbitrary indexer/tracker host.
headers: dict[str, str] = {"Accept": "application/x-bittorrent"}
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
@@ -178,47 +114,44 @@ def _fetch_torrent_info(url: str) -> TorrentInfo:
try:
logger.debug("Fetching torrent file from: %s...", url[:80])
# Redirects are followed manually: some indexers redirect download URLs
# to magnet links, and each hop must decide anew whether it may see the
# API key.
current_url = url
redirects_remaining = _TORRENT_FETCH_MAX_REDIRECTS
while True:
request_headers = dict(headers)
if not _is_trusted_torrent_fetch_url(current_url):
request_headers.pop("X-Api-Key", None)
# Use allow_redirects=False to handle magnet link redirects manually
# Some indexers redirect download URLs to magnet links
resp = requests.get(
url,
timeout=30,
allow_redirects=False,
headers=headers,
verify=get_ssl_verify(url),
)
resp = requests.get(
current_url,
timeout=30,
allow_redirects=False,
headers=request_headers,
verify=get_ssl_verify(current_url),
)
if resp.status_code not in (301, 302, 303, 307, 308):
break
redirect_url = resolve_url(current_url, resp.headers.get("Location", ""))
# Check if this is a redirect to a magnet link
if resp.status_code in (301, 302, 303, 307, 308):
redirect_url = resolve_url(url, resp.headers.get("Location", ""))
if redirect_url.startswith("magnet:"):
logger.debug("Download URL redirected to magnet link")
info_hash = extract_hash_from_magnet(redirect_url)
if not info_hash and expected_hash:
info_hash = expected_hash
return TorrentInfo(
info_hash=extract_hash_from_magnet(redirect_url),
info_hash=info_hash,
torrent_data=None,
is_magnet=True,
magnet_url=redirect_url,
)
if redirects_remaining <= 0:
logger.warning("Too many redirects fetching torrent file: %s...", url[:80])
return TorrentInfo(
info_hash=None,
torrent_data=None,
is_magnet=False,
fetch_error="too many redirects",
if not _is_trusted_torrent_fetch_url(redirect_url):
logger.debug(
"Skipping torrent prefetch redirect to untrusted URL: %s...",
redirect_url[:80],
)
redirects_remaining -= 1
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
# Not a magnet redirect, follow it manually
logger.debug("Following redirect to: %s...", redirect_url[:80])
current_url = redirect_url
resp = requests.get(
redirect_url,
timeout=30,
headers=headers,
verify=get_ssl_verify(redirect_url),
)
resp.raise_for_status()
torrent_data = resp.content
@@ -229,22 +162,25 @@ def _fetch_torrent_info(url: str) -> TorrentInfo:
text_content = torrent_data.decode("utf-8", errors="ignore").strip()
if text_content.startswith("magnet:"):
logger.debug("Download URL returned magnet link as response body")
info_hash = extract_hash_from_magnet(text_content)
if not info_hash and expected_hash:
info_hash = expected_hash
return TorrentInfo(
info_hash=extract_hash_from_magnet(text_content),
info_hash=info_hash,
torrent_data=None,
is_magnet=True,
magnet_url=text_content,
)
info_hash = extract_info_hash_from_torrent(torrent_data)
info_hash = extract_info_hash_from_torrent(torrent_data) or expected_hash
if info_hash:
logger.debug("Extracted hash from torrent file: %s", info_hash)
else:
logger.warning("Could not extract hash from torrent file")
return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False)
except _TORRENT_FETCH_ERRORS as e:
logger.warning("Could not fetch torrent file: %s", e)
return TorrentInfo(info_hash=None, torrent_data=None, is_magnet=False, fetch_error=str(e))
logger.debug("Could not fetch torrent file: %s", e)
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
def _is_trusted_torrent_fetch_url(url: str) -> bool:
+2 -30
View File
@@ -316,12 +316,8 @@ class TransmissionClient(DownloadClient):
state, message = status_map.get(status_value, ("downloading", "Downloading"))
progress = torrent.percent_done * 100
# Only mark complete when seeding or stopped (e.g. if seed limit/ratio is 0)
# and progress is complete. seed pending means files still being moved
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value in (
"seeding",
"stopped",
)
# Only mark complete when seeding - seed pending means files still being moved
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value == "seeding"
if complete:
message = "Complete"
@@ -390,30 +386,6 @@ class TransmissionClient(DownloadClient):
else:
return True
def _get_torrent_labels(self, download_id: str) -> list[str]:
"""Return a torrent's current labels, preserving their order."""
torrent = self._client.get_torrent(download_id)
raw_labels = getattr(torrent, "labels", None) or []
return [str(label) for label in raw_labels if str(label)]
def set_category(self, download_id: str, category: str) -> bool:
"""Add the post-import label to a torrent, keeping labels set elsewhere."""
try:
existing_labels = self._get_torrent_labels(download_id)
if category in existing_labels:
logger.debug(
"Transmission torrent %s already has label '%s'", download_id, category
)
return True
self._client.change_torrent(ids=download_id, labels=[*existing_labels, category])
logger.info("Added Transmission label '%s' to %s", category, download_id)
except _TRANSMISSION_CLIENT_ERRORS as e:
self._log_error("set_category", e)
return False
else:
return True
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where torrent files are located.
+6 -153
View File
@@ -4,7 +4,6 @@ These utilities handle file collisions atomically, avoiding TOCTOU race conditio
when multiple workers may try to write to the same path simultaneously.
"""
import contextlib
import errno
import os
import shutil
@@ -105,57 +104,6 @@ _PUBLISH_VERIFY_RETRY_SECONDS = 0.25
_TEMPFILE_PREFIX = ".shelfmark."
_TEMPFILE_SUFFIX = ".tmp"
# Destinations that accept writes but reject unlink/rename, e.g. a Synology share
# with "Delete subfolders and files" unticked. Publishing a temp file into place
# removes a directory entry, so those paths must be written in place instead.
_DELETE_DENIED_DIRS: set[str] = set()
class _PublishDeniedError(Exception):
"""A fully-written temp file could not be renamed onto its final path."""
def _is_delete_denied_error(error: Exception) -> bool:
return isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM}
def mark_delete_denied(directory: Path) -> None:
"""Record that `directory` rejects deletes so later writes skip the temp file."""
key = str(directory)
if key in _DELETE_DENIED_DIRS:
return
_DELETE_DENIED_DIRS.add(key)
logger.warning(
"Destination %s rejects delete/rename; writing files in place instead of "
"publishing atomically. Grant delete permission to restore atomic writes.",
directory,
)
def clear_delete_denied(directory: Path) -> None:
"""Forget recorded denials for `directory` and anything beneath it.
Subdirectories get marked independently (an `organize` layout publishes into
per-author folders), so clearing only the exact key would leave a fixed
destination writing in place until restart.
"""
if not _DELETE_DENIED_DIRS:
return
key = str(directory)
prefix = f"{key}{os.sep}"
_DELETE_DENIED_DIRS.difference_update(
{marked for marked in _DELETE_DENIED_DIRS if marked == key or marked.startswith(prefix)}
)
def is_delete_denied(directory: Path) -> bool:
"""True if `directory` or one of its ancestors is known to reject deletes."""
if not _DELETE_DENIED_DIRS:
return False
if str(directory) in _DELETE_DENIED_DIRS:
return True
return any(str(parent) in _DELETE_DENIED_DIRS for parent in directory.parents)
def _verify_transfer_size(
dest: Path,
@@ -413,40 +361,10 @@ def _create_temp_path(dest_path: Path) -> Path:
return Path(temp_path)
def _discard_path(path: Path) -> None:
"""Best-effort unlink that tolerates destinations which reject deletes."""
try:
run_blocking_io(path.unlink, missing_ok=True)
except OSError as exc:
logger.warning("Could not remove %s: %s", path, exc)
def _copy_into_claimed(source_path: Path, dest_path: Path, expected_size: int) -> None:
"""Copy content straight into an already-claimed destination path.
Used when the destination rejects rename/unlink: there is no temp file to
publish, so the final name is written in place. This is not atomic - a
watcher can observe a partial file - but it is the only way to deliver on
such a share. `copyfile` (not `copy2`) because metadata copying needs chmod,
which those shares also tend to refuse.
"""
try:
run_blocking_io(shutil.copyfile, str(source_path), str(dest_path))
_verify_transfer_size(dest_path, expected_size, "copy")
except Exception:
with contextlib.suppress(OSError):
run_blocking_io(dest_path.unlink, missing_ok=True)
raise
def _publish_temp_file(temp_path: Path, dest_path: Path) -> bool:
"""Publish a temp file to its final path without overwriting existing files.
Returns True on success, False if the destination already exists.
Raises `_PublishDeniedError` when the rename is refused for lack of delete
permission. The claimed destination is left in place so the caller can write
into it directly instead.
"""
claimed = _claim_destination(dest_path)
if not claimed:
@@ -456,19 +374,7 @@ def _publish_temp_file(temp_path: Path, dest_path: Path) -> bool:
# Publish by renaming the fully-written temp file into place. This gives
# watchers an IN_MOVED_TO-style event on the final path instead of relying
# on hardlink support in the destination filesystem.
try:
run_blocking_io(os.replace, str(temp_path), str(dest_path))
except OSError as e:
if _is_delete_denied_error(e):
log_transfer_permission_context(
"publish_replace",
source=temp_path,
dest=dest_path,
error=e,
)
mark_delete_denied(dest_path.parent)
raise _PublishDeniedError(str(e)) from e
raise
run_blocking_io(os.replace, str(temp_path), str(dest_path))
# Best-effort nudge for watchers that only react to close-write on the
# final filename rather than rename/move events.
@@ -477,8 +383,6 @@ def _publish_temp_file(temp_path: Path, dest_path: Path) -> bool:
run_blocking_io(os.close, fd)
except OSError:
pass
except _PublishDeniedError:
raise
except Exception as e:
if _is_permission_error(e):
log_transfer_permission_context(
@@ -487,23 +391,12 @@ def _publish_temp_file(temp_path: Path, dest_path: Path) -> bool:
dest=dest_path,
error=e,
)
_discard_path(dest_path)
run_blocking_io(dest_path.unlink, missing_ok=True)
raise
else:
return True
def _move_via_copy(source_path: Path, dest_path: Path, max_attempts: int) -> Path:
"""Deliver a move as copy + source unlink.
For destinations that reject rename. The source lives in TMP_DIR (which we
own and can delete), so only the destination-side semantics change.
"""
final_path = atomic_copy(source_path, dest_path, max_attempts=max_attempts)
_discard_path(source_path)
return final_path
def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Move a file with collision detection.
@@ -530,11 +423,6 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
ext = dest_path.suffix
parent = dest_path.parent
# rename() removes a directory entry, so a destination that refuses deletes
# cannot be moved into. Deliver it as copy + source unlink instead.
if is_delete_denied(parent):
return _move_via_copy(source_path, dest_path, max_attempts)
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
@@ -561,13 +449,6 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
run_blocking_io(try_path.unlink, missing_ok=True)
continue
except OSError as e:
if _is_delete_denied_error(e):
# Destination refuses the rename; fall back to copy + unlink source.
mark_delete_denied(parent)
if claimed:
_discard_path(try_path)
return _move_via_copy(source_path, dest_path, max_attempts)
# Cross-filesystem - copy to temp and publish atomically.
if e.errno != errno.EXDEV:
if claimed:
@@ -616,7 +497,7 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
try:
_verify_published_file(try_path, expected_size, "move")
except Exception:
_discard_path(try_path)
run_blocking_io(try_path.unlink, missing_ok=True)
raise
run_blocking_io(source_path.unlink)
@@ -627,18 +508,9 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
if temp_path:
run_blocking_io(temp_path.unlink, missing_ok=True)
continue
except _PublishDeniedError:
# Destination is claimed but unrenameable; write into it directly.
_copy_into_claimed(source_path, try_path, expected_size)
if temp_path:
_discard_path(temp_path)
_discard_path(source_path)
if attempt > 0:
logger.info("File collision resolved: %s", try_path.name)
return try_path
except Exception:
if temp_path:
_discard_path(temp_path)
run_blocking_io(temp_path.unlink, missing_ok=True)
raise
else:
return try_path
@@ -757,17 +629,6 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
if run_blocking_io(try_path.exists):
continue
# Known-undeletable destination: skip the temp file entirely, otherwise
# every transfer would strand a `.shelfmark.*.tmp` we cannot clean up.
if is_delete_denied(parent):
if not _claim_destination(try_path):
continue
_copy_into_claimed(source_path, try_path, expected_size)
if attempt > 0:
logger.info("File collision resolved: %s", try_path.name)
return try_path
temp_path: Path | None = None
try:
temp_path = _create_temp_path(try_path)
@@ -819,22 +680,14 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
try:
_verify_published_file(try_path, expected_size, "copy")
except Exception:
_discard_path(try_path)
run_blocking_io(try_path.unlink, missing_ok=True)
raise
if attempt > 0:
logger.info("File collision resolved: %s", try_path.name)
except _PublishDeniedError:
# The destination is claimed but unrenameable; write into it directly.
_copy_into_claimed(source_path, try_path, expected_size)
if temp_path:
_discard_path(temp_path)
if attempt > 0:
logger.info("File collision resolved: %s", try_path.name)
return try_path
except Exception:
if temp_path:
_discard_path(temp_path)
run_blocking_io(temp_path.unlink, missing_ok=True)
raise
else:
return try_path
+23 -74
View File
@@ -4,6 +4,7 @@ import random
import time
from http import HTTPStatus
from io import BytesIO
from threading import Event, Thread
from typing import TYPE_CHECKING, NoReturn
from urllib.parse import urljoin, urlparse
@@ -15,34 +16,24 @@ from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int
from shelfmark.download import network
from shelfmark.download.activity import release_activity_grace, request_activity_grace
from shelfmark.download.network import get_proxies, get_ssl_verify
if TYPE_CHECKING:
from collections.abc import Callable
from threading import Event
from types import ModuleType
logger = setup_logger(__name__)
_RNG = random.SystemRandom()
_MAX_REDIRECTS = 5
# Z-Library answers the first hit with a 503 whose only real payload is a Set-Cookie; echoing
# that cookie back returns the 302 to the real page. Two attempts cover the handshake without
# letting a server that keeps re-issuing cookies hold us in the loop.
_MAX_COOKIE_HANDSHAKE_RETRIES = 2
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
_HTTP_STATUS_RATE_LIMITED = HTTPStatus.TOO_MANY_REQUESTS
_HTTP_STATUS_SERVICE_UNAVAILABLE = HTTPStatus.SERVICE_UNAVAILABLE
_HTTP_STATUS_OK = HTTPStatus.OK
_HTTP_STATUS_RANGE_NOT_SATISFIABLE = HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE
_HTTP_STATUS_PARTIAL_CONTENT = HTTPStatus.PARTIAL_CONTENT
_HTTP_STATUS_NON_RETRYABLE = (_HTTP_STATUS_FORBIDDEN, _HTTP_STATUS_NOT_FOUND)
_STATUS_CALLBACK_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError)
# Added on top of the active bypasser's own budget so it reports its real failure before
# stall detection cancels the download.
_BYPASS_GRACE_SLACK_SECONDS = 30.0
_BYPASSER_ERRORS = (
AttributeError,
BypassCancelledError,
@@ -63,18 +54,6 @@ def _raise_too_many_redirects(message: str) -> NoReturn:
raise requests.exceptions.TooManyRedirects(message)
def _new_cookies(response: requests.Response, already_sent: dict[str, str]) -> dict[str, str]:
"""Cookies a response set that we were not already echoing back.
Returning only the *new* ones is what makes the retry terminate: a server that keeps
re-issuing the same cookie yields nothing here, so we stop instead of spinning.
"""
jar = getattr(response, "cookies", None)
if not jar:
return {}
return {name: value for name, value in jar.items() if already_sent.get(name) != value}
def _get_internal_bypasser() -> ModuleType:
"""Lazy import of internal bypasser module."""
global _internal_bypasser
@@ -120,19 +99,6 @@ def _is_cf_bypass_enabled() -> bool:
return coerce_bool(app_config.get("USE_CF_BYPASS", True))
def _bypass_grace_seconds() -> float:
"""How long a bypass may block before stall detection should give up on it.
Each bypasser knows its own retry/timeout budget, so ask the active one rather than
duplicating the arithmetic here. The slack keeps the bypasser's own deadline expiring
first, so the user sees its real error instead of a generic "Download stalled".
"""
bypasser = (
_get_external_bypasser() if _is_using_external_bypasser() else _get_internal_bypasser()
)
return bypasser.max_duration_seconds() + _BYPASS_GRACE_SLACK_SECONDS
def get_bypassed_page(
url: str,
selector: network.AAMirrorSelector | None = None,
@@ -295,9 +261,6 @@ def html_get_page(
original_url = url
current_url = selector.rewrite(original_url)
use_bypasser_now = use_bypasser
# Survives across attempts so a cookie won once is still presented on later retries.
handshake_cookies: dict[str, str] = {}
handshake_retries = 0
for attempt in range(1, retry_limit + 1):
# Check for cancellation before each attempt
@@ -310,27 +273,34 @@ def html_get_page(
if use_bypasser_now and _is_cf_bypass_enabled():
if status_callback:
status_callback("resolving", "Bypassing protection...")
heartbeat_stop = Event()
heartbeat_thread: Thread | None = None
if status_callback:
def _heartbeat() -> None:
# Keep the download "alive" during long bypass operations so the orchestrator
# doesn't flag it as stalled.
if cancel_flag and cancel_flag.is_set():
return
try:
status_callback("resolving", "Bypassing protection...")
except _STATUS_CALLBACK_ERRORS:
return
heartbeat_thread = Thread(
target=_heartbeat, daemon=True, name="BypassHeartbeat"
)
heartbeat_thread.start()
try:
# A bypass is one long blocking call with no incremental progress, so
# tell the orchestrator up front how long it may legitimately take
# instead of trying to fake activity while it runs. Inside the try so a
# bypasser that fails to load is still reported as a bypasser error.
request_activity_grace(status_callback, _bypass_grace_seconds())
result = get_bypassed_page(current_url, selector, cancel_flag)
return _result(result or "", current_url)
except _BYPASSER_ERRORS as e:
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
# Surface the real reason. Without this the caller only sees an empty
# page and the download dies with a generic failure, hiding e.g. a
# FlareSolverr 500 behind a silent wait.
if status_callback and not isinstance(e, BypassCancelledError):
try:
status_callback("error", f"Bypass failed: {type(e).__name__}: {e}")
except _STATUS_CALLBACK_ERRORS:
logger.debug("Bypass error status callback failed", exc_info=True)
return _result("", current_url)
finally:
release_activity_grace(status_callback)
heartbeat_stop.set()
if heartbeat_thread:
heartbeat_thread.join(timeout=1)
logger.debug("GET: %s", current_url)
@@ -352,31 +322,12 @@ def html_get_page(
current_url,
proxies=get_proxies(current_url),
timeout=REQUEST_TIMEOUT,
# Bypasser-derived cookies win: they came from a real solved challenge.
cookies={**handshake_cookies, **cookies},
cookies=cookies,
headers=headers,
allow_redirects=allow_redirects,
verify=get_ssl_verify(current_url),
)
# Z-Library gates the first hit with a 503 that carries nothing but a
# Set-Cookie; echoing it back yields the 302 to the real page. Without this
# the cookie is dropped and every retry re-runs the same rejected request.
if (
response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE
and handshake_retries < _MAX_COOKIE_HANDSHAKE_RETRIES
):
issued = _new_cookies(response, handshake_cookies)
if issued:
handshake_cookies.update(issued)
handshake_retries += 1
logger.debug(
"503 set %s cookie(s); retrying with them: %s",
len(issued),
current_url,
)
continue
if is_aa_url and response.is_redirect:
location = response.headers.get("Location", "")
if not location:
@@ -405,7 +356,6 @@ def html_get_page(
current_url = new_url
# Reset per-request state for the new host.
headers = {"User-Agent": DOWNLOAD_HEADERS["User-Agent"]}
handshake_cookies.clear()
is_aa_url = network.should_rotate_dns_for_url(current_url)
allow_redirects = not is_aa_url
redirects_followed = 0
@@ -475,7 +425,6 @@ def html_get_page(
new_url = _try_rotation(original_url, current_url, selector)
if new_url:
current_url = new_url
handshake_cookies.clear()
continue
# Retry with backoff
-93
View File
@@ -852,99 +852,6 @@ def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver:
return custom_resolver
# --- ISP / network DNS interference detection ---------------------------------
# Compare what the (tamperable) system resolver returns for a host against a
# tamper-resistant DoH lookup. Divergent answers are a strong signal the network is
# hijacking or NXDOMAIN-blocking the domain (a common reason AA downloads "work" but
# land on an ISP block page). Used to surface an actionable hint to the user.
_dns_interference_warned: set[str] = set()
_dns_interference_active = False
def _build_detection_doh_resolver() -> DoHResolver | None:
"""Build a throwaway DoH resolver for interference checks (no socket patching).
Honours the DoH provider the user selected (``DNS_PROVIDERS[_current_dns_index]``),
falling back to the first configured provider when none is active. The endpoint is
pinned to the provider's own nameserver IP so resolving the DoH host can't be
redirected by the very DNS layer the check is meant to detect.
"""
if 0 <= _current_dns_index < len(DNS_PROVIDERS):
_name, servers, doh_url = DNS_PROVIDERS[_current_dns_index]
elif DNS_PROVIDERS:
_name, servers, doh_url = DNS_PROVIDERS[0]
else:
return None
server_hostname = urllib.parse.urlparse(doh_url).hostname or ""
if not server_hostname or not servers:
return None
return DoHResolver(doh_url, server_hostname, servers[0])
def detect_dns_interference(hostname: str) -> dict[str, list[str]] | None:
"""Detect network DNS interference by comparing system DNS against DoH.
Returns ``{"system_ips": [...], "doh_ips": [...]}`` when the two resolvers disagree
(no overlapping IPs), otherwise None. No-op for IP literals / local hostnames and
when DoH resolution is unavailable, so it never produces a false positive.
"""
host = (hostname or "").strip().lower()
if not host or _is_ip_address(host) or _is_local_address(host):
return None
resolver = _build_detection_doh_resolver()
if resolver is None:
return None
try:
system_ips = {str(info[4][0]) for info in original_getaddrinfo(host, 443, socket.AF_INET)}
except OSError:
return None
if not system_ips:
return None
doh_ips = {ip for ip in resolver.resolve(host, "A") if ip}
if not doh_ips or (system_ips & doh_ips):
return None
return {"system_ips": sorted(system_ips), "doh_ips": sorted(doh_ips)}
def note_possible_dns_interference(hostname: str) -> bool:
"""Check ``hostname`` for DNS interference, logging an actionable warning once.
Returns True when interference has been detected this session. The check runs at
most once per host to avoid repeated DoH lookups and log spam.
"""
global _dns_interference_active
host = (hostname or "").strip().lower()
if not host or host in _dns_interference_warned:
return _dns_interference_active
_dns_interference_warned.add(host)
result = detect_dns_interference(host)
if not result:
return _dns_interference_active
_dns_interference_active = True
routing_via_doh = _current_dns_index >= 0 and bool(DOH_SERVER)
remedy = (
"Shelfmark is routing this domain through DNS-over-HTTPS to work around it."
if routing_via_doh
else "Enable DNS-over-HTTPS (USE_DOH=true) or set a custom DNS provider to bypass it."
)
logger.warning(
"Possible ISP/network DNS interference for %s: system DNS resolves to %s but DoH "
"resolves to %s. The network appears to be blocking or redirecting this domain. %s",
host,
result["system_ips"],
result["doh_ips"],
remedy,
)
return True
def dns_interference_detected() -> bool:
"""Whether network DNS interference has been detected this session."""
return _dns_interference_active
def init_doh_resolver(doh_server: str = "") -> DoHResolver | None:
"""Initialize DNS over HTTPS resolver."""
server = doh_server or DOH_SERVER
+23 -135
View File
@@ -12,7 +12,7 @@ from concurrent.futures import Future, ThreadPoolExecutor
from email.utils import parseaddr
from pathlib import Path
from threading import Event, Lock
from typing import TYPE_CHECKING, Any
from typing import Any
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
@@ -24,7 +24,6 @@ from shelfmark.core.request_helpers import (
)
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.core.utils import transform_cover_url
from shelfmark.download.activity import parse_activity_grace
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.postprocess.pipeline import is_torrent_source, safe_cleanup_path
from shelfmark.download.postprocess.router import post_process_download
@@ -34,9 +33,6 @@ from shelfmark.release_sources import (
get_source_display_name,
)
if TYPE_CHECKING:
from collections.abc import Iterable
logger = setup_logger(__name__)
_RNG = random.SystemRandom()
@@ -69,16 +65,7 @@ _last_progress_value: dict[str, float] = {}
# De-duplicate status updates (keep-alive updates shouldn't spam clients)
_last_status_event: dict[str, tuple[str, str | None]] = {}
STALL_TIMEOUT = 300 # 5 minutes without progress/status update = stalled
# Absolute deadlines (time.time()) until which stall detection is suppressed for a task.
# Long single-shot operations (protection bypass, etc.) declare their own upper bound via
# `shelfmark.download.activity` instead of faking progress. See set_activity_grace().
_activity_grace: dict[str, float] = {}
# A caller cannot buy immortality: the largest grace any operation may request. Must stay
# above the largest budget any caller can declare (see http._bypass_grace_seconds).
_MAX_ACTIVITY_GRACE_SECONDS = 960.0
COORDINATOR_LOOP_ERROR_RETRY_DELAY = 1.0
# Ceiling for the exponential backoff applied to repeated coordinator loop failures.
_COORDINATOR_LOOP_ERROR_MAX_DELAY = 30.0
_PROGRESS_BROADCAST_START_PERCENT = 1
_PROGRESS_BROADCAST_COMPLETE_PERCENT = 99
_PROGRESS_BROADCAST_MIN_DELTA = 10
@@ -183,19 +170,16 @@ def _build_retry_resolution_fields(
retry_download_url = normalize_optional_text(release_data.get("download_url"))
protocol = normalize_optional_text(release_data.get("protocol"))
source = normalize_optional_text(release_data.get("source"))
retry_source_context: dict[str, Any] = {}
if source is not None:
handler = get_handler(source)
source_retry_fields = handler.build_retry_resolution_fields(release_data)
if "retry_download_url" in source_retry_fields:
retry_download_url = normalize_optional_text(
source_retry_fields.get("retry_download_url")
)
if "retry_download_protocol" in source_retry_fields:
protocol = normalize_optional_text(source_retry_fields.get("retry_download_protocol"))
raw_retry_source_context = source_retry_fields.get("retry_source_context")
if isinstance(raw_retry_source_context, dict):
retry_source_context = dict(raw_retry_source_context)
retry_download_url = (
normalize_optional_text(source_retry_fields.get("retry_download_url"))
or retry_download_url
)
protocol = (
normalize_optional_text(source_retry_fields.get("retry_download_protocol")) or protocol
)
ratio_limit = _optional_number(release_data.get("ratio_limit"))
if ratio_limit is None and config.get("PROWLARR_USE_SEED_PREFERENCES", False):
@@ -218,7 +202,6 @@ def _build_retry_resolution_fields(
),
"retry_ratio_limit": ratio_limit,
"retry_seeding_time_limit_minutes": seeding_time_limit_minutes,
"retry_source_context": retry_source_context,
"can_retry_without_staged_source": True,
}
@@ -264,7 +247,6 @@ def queue_release(
series_name = release_data.get("series_name") or extra.get("series_name")
series_position = release_data.get("series_position") or extra.get("series_position")
subtitle = release_data.get("subtitle") or extra.get("subtitle")
language = release_data.get("language") or extra.get("language")
books_output_mode = (
str(config.get("BOOKS_OUTPUT_MODE", "folder", user_id=user_id) or "folder")
@@ -299,7 +281,6 @@ def queue_release(
series_name=series_name,
series_position=series_position,
subtitle=subtitle,
language=language,
search_mode=search_mode,
output_mode=output_mode,
output_args=output_args,
@@ -419,7 +400,6 @@ def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
search_mode = normalized_search_mode or None
raw_output_args = getattr(task, "output_args", None)
raw_retry_source_context = getattr(task, "retry_source_context", None)
return {
"task_id": getattr(task, "task_id", None),
@@ -435,7 +415,6 @@ def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
"series_name": getattr(task, "series_name", None),
"series_position": getattr(task, "series_position", None),
"subtitle": getattr(task, "subtitle", None),
"language": getattr(task, "language", None),
"search_mode": search_mode,
"output_mode": getattr(task, "output_mode", None),
"output_args": dict(raw_output_args) if isinstance(raw_output_args, dict) else {},
@@ -449,9 +428,6 @@ def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
"retry_expected_hash": getattr(task, "retry_expected_hash", None),
"retry_ratio_limit": getattr(task, "retry_ratio_limit", None),
"retry_seeding_time_limit_minutes": getattr(task, "retry_seeding_time_limit_minutes", None),
"retry_source_context": (
dict(raw_retry_source_context) if isinstance(raw_retry_source_context, dict) else {}
),
"can_retry_without_staged_source": bool(
getattr(task, "can_retry_without_staged_source", True)
),
@@ -477,7 +453,6 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
search_mode = None
output_args = payload.get("output_args")
retry_source_context = payload.get("retry_source_context")
return DownloadTask(
task_id=task_id,
@@ -493,7 +468,6 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
series_name=normalize_optional_text(payload.get("series_name")),
series_position=_optional_number(payload.get("series_position")),
subtitle=normalize_optional_text(payload.get("subtitle")),
language=normalize_optional_text(payload.get("language")),
search_mode=search_mode,
output_mode=normalize_optional_text(payload.get("output_mode")),
output_args=dict(output_args) if isinstance(output_args, dict) else {},
@@ -509,9 +483,6 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
retry_seeding_time_limit_minutes=_optional_positive_int(
payload.get("retry_seeding_time_limit_minutes")
),
retry_source_context=(
dict(retry_source_context) if isinstance(retry_source_context, dict) else {}
),
can_retry_without_staged_source=bool(payload.get("can_retry_without_staged_source", True)),
)
@@ -662,17 +633,6 @@ def _download_task(task_id: str, cancel_flag: Event) -> str | None:
update_download_progress(task_id, progress)
def status_callback(status: str, message: str | None = None) -> None:
# Liveness hint from a long single-shot operation, not a user-visible status.
# Handled here so it never reaches update_download_status (which dedupes status
# transitions on purpose). See shelfmark.download.activity.
grace = parse_activity_grace(status, message)
if grace is not None:
if grace > 0:
set_activity_grace(task_id, grace)
else:
clear_activity_grace(task_id)
return
status_key = status.lower()
if status_key == "error":
_capture_task_error(
@@ -910,7 +870,6 @@ def _cleanup_progress_tracking(task_id: str) -> None:
_last_activity.pop(task_id, None)
_last_progress_value.pop(task_id, None)
_last_status_event.pop(task_id, None)
_activity_grace.pop(task_id, None)
def _finalize_download_failure(task_id: str) -> None:
@@ -958,66 +917,6 @@ def _process_single_download(task_id: str, cancel_flag: Event) -> None:
ws_manager.broadcast_status_update(queue_status())
def set_activity_grace(book_id: str, seconds: float) -> None:
"""Suppress stall detection for `book_id` for up to `seconds` from now.
For long single-shot operations that cannot report incremental progress (protection
bypass being the motivating case). The grace is a single absolute deadline computed
once, so it cannot be extended into immortality by a keep-alive that carries no real
liveness information - an operation that hangs forever is still cancelled once its
declared budget expires.
Deliberately touches neither the queue nor the WebSocket: this is a liveness
assertion, not a user-visible status transition.
"""
grace = _config_float(seconds, 0.0)
grace = min(max(grace, 0.0), _MAX_ACTIVITY_GRACE_SECONDS)
with _progress_lock:
_activity_grace[book_id] = time.time() + grace
def clear_activity_grace(book_id: str) -> None:
"""Drop any activity grace for `book_id` and count this moment as activity.
Resetting `_last_activity` means a nested or abandoned grace degrades to a fresh
full STALL_TIMEOUT window rather than an immediate stall.
"""
with _progress_lock:
_activity_grace.pop(book_id, None)
_last_activity[book_id] = time.time()
def _find_stalled_tasks(task_ids: Iterable[str], now: float) -> list[str]:
"""Return the task ids with no activity inside STALL_TIMEOUT and no active grace.
Holds `_progress_lock` for dict reads only - never call into `book_queue` from here,
see _cancel_stalled_task().
"""
stalled: list[str] = []
with _progress_lock:
for task_id in task_ids:
last_active = _last_activity.get(task_id, now)
deadline = max(last_active + STALL_TIMEOUT, _activity_grace.get(task_id, 0.0))
if now > deadline:
stalled.append(task_id)
return stalled
def _cancel_stalled_task(task_id: str) -> None:
"""Cancel a stalled download.
Must be called WITHOUT `_progress_lock` held. `book_queue.cancel_download` runs the
terminal-status hooks, which reach a sqlite write that gevent does not patch; holding
the progress lock across that blocks the hub and every other download worker.
"""
logger.warning("Download stalled for %s, cancelling", task_id)
book_queue.cancel_download(task_id)
book_queue.update_status_message(
task_id,
f"Download stalled (no activity for {STALL_TIMEOUT}s)",
)
def concurrent_download_loop() -> None:
"""Run the main concurrent download coordinator."""
max_workers = normalize_positive_int(config.MAX_CONCURRENT_DOWNLOADS) or 1
@@ -1027,7 +926,6 @@ def concurrent_download_loop() -> None:
with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor:
active_futures: dict[Future, tuple[str, Event]] = {} # Track active download futures
stalled_tasks: set[str] = set() # Track tasks already cancelled due to stall
consecutive_errors = 0
while True:
try:
@@ -1078,14 +976,19 @@ def concurrent_download_loop() -> None:
# Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
current_time = time.time()
candidates = [
task_id
for _future, (task_id, _cancel_flag) in list(active_futures.items())
if task_id not in stalled_tasks
]
for task_id in _find_stalled_tasks(candidates, current_time):
_cancel_stalled_task(task_id)
stalled_tasks.add(task_id)
with _progress_lock:
for _future, (task_id, _cancel_flag) in list(active_futures.items()):
if task_id in stalled_tasks:
continue
last_active = _last_activity.get(task_id, current_time)
if current_time - last_active > STALL_TIMEOUT:
logger.warning("Download stalled for %s, cancelling", task_id)
book_queue.cancel_download(task_id)
book_queue.update_status_message(
task_id,
f"Download stalled (no activity for {STALL_TIMEOUT}s)",
)
stalled_tasks.add(task_id)
# Start new downloads if we have capacity
while len(active_futures) < max_workers:
@@ -1108,24 +1011,9 @@ def concurrent_download_loop() -> None:
# Brief sleep to prevent busy waiting
time.sleep(main_loop_sleep_time)
consecutive_errors = 0
# This loop is the only thing driving the download queue; if it exits, nothing
# is ever picked up again and the app looks healthy while doing nothing (#823,
# #1166). A narrow exception list let gevent's LoopExit and friends through, so
# catch everything short of BaseException - GreenletExit and gevent.Timeout must
# still propagate, and the tests' loop-stopping sentinels derive from
# BaseException for exactly this reason.
except Exception as e: # noqa: BLE001 - coordinator loop must never die
consecutive_errors += 1
except (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError) as e:
logger.error_trace("Download coordinator loop error: %s", e)
# Back off when the failure is persistent so we don't spin at 1Hz forever,
# but keep the first delay unchanged for a normal transient blip.
time.sleep(
min(
COORDINATOR_LOOP_ERROR_RETRY_DELAY * 2 ** min(consecutive_errors - 1, 5),
_COORDINATOR_LOOP_ERROR_MAX_DELAY,
)
)
time.sleep(COORDINATOR_LOOP_ERROR_RETRY_DELAY)
# Download coordinator thread (started explicitly via start())
@@ -229,7 +229,6 @@ def _build_custom_script_payload(
"series_name": context.task.series_name,
"series_position": context.task.series_position,
"subtitle": context.task.subtitle,
"language": context.task.language,
"original_download_path": context.task.original_download_path,
},
"output": {
+4 -33
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
import contextlib
import uuid
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
@@ -12,11 +12,7 @@ from shelfmark.core.utils import (
from shelfmark.core.utils import (
is_audiobook as check_audiobook,
)
from shelfmark.download.fs import (
clear_delete_denied,
mark_delete_denied,
run_blocking_io,
)
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.permissions_debug import log_path_permission_context
from shelfmark.release_sources import get_source
@@ -28,8 +24,6 @@ if TYPE_CHECKING:
logger = setup_logger("shelfmark.download.postprocess.pipeline")
_WRITE_PROBE_NAME = ".shelfmark_write_test.tmp"
def validate_destination(
destination: Path, status_callback: Callable[[str, str | None], None]
@@ -46,20 +40,16 @@ def validate_destination(
status_callback("error", f"Destination is not a directory: {destination}")
return False
created_by_us = False
if not destination_exists:
try:
run_blocking_io(destination.mkdir, parents=True, exist_ok=True)
created_by_us = True
except (OSError, PermissionError) as exc:
log_path_permission_context("destination_create", destination)
logger.warning("Cannot create destination: %s (%s)", destination, exc)
status_callback("error", f"Cannot create destination: {destination} ({exc})")
return False
# Stable name: on shares that refuse deletes the probe file cannot be cleaned
# up, so reusing one name bounds the leftovers at a single hidden file.
test_path = destination / _WRITE_PROBE_NAME
test_path = destination / f".shelfmark_write_test_{uuid.uuid4().hex}.tmp"
try:
test_content = (
@@ -67,33 +57,14 @@ def validate_destination(
"It should've been automatically deleted. Feel free to delete it.\n"
)
run_blocking_io(test_path.write_text, test_content)
run_blocking_io(test_path.unlink, missing_ok=True)
except OSError as exc:
logger.debug("Destination write probe path: %s", test_path)
log_path_permission_context("destination_write_probe", destination)
logger.warning("Destination not writable: %s (%s)", destination, exc)
status_callback("error", f"Destination not writable: {destination} ({exc})")
if created_by_us:
with contextlib.suppress(OSError):
run_blocking_io(destination.rmdir)
return False
try:
run_blocking_io(test_path.unlink, missing_ok=True)
except OSError as exc:
# Writable but not deletable, e.g. a Synology share with "Delete
# subfolders and files" unticked. Not fatal: record it so transfers write
# files in place instead of publishing a temp file via rename.
mark_delete_denied(destination)
logger.warning(
"Destination %s is writable but refuses deletes (%s); leaving probe file %s "
"behind and writing files in place",
destination,
exc,
test_path.name,
)
else:
clear_delete_denied(destination)
return True
@@ -12,7 +12,6 @@ from shelfmark.core.naming import (
assign_part_numbers,
build_library_path,
derive_primary_title,
normalize_language_code,
parse_naming_template,
sanitize_filename,
)
@@ -64,7 +63,6 @@ def build_metadata_dict(task: DownloadTask) -> dict:
"Year": task.year,
"Series": task.series_name,
"SeriesPosition": task.series_position,
"Language": normalize_language_code(task.language),
"User": task.username,
}
+1 -4
View File
@@ -118,7 +118,7 @@ BASE_PATH = normalize_base_path(normalize_optional_text(app_config.get("URL_BASE
app = Flask(__name__)
app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 0 # Disable caching
app.config["APPLICATION_ROOT"] = BASE_PATH or "/"
wsgi_app = cast(Any, ProxyFix(app.wsgi_app, x_host=1, x_port=1))
wsgi_app = cast(Any, ProxyFix(app.wsgi_app))
if BASE_PATH:
wsgi_app = cast(Any, PrefixMiddleware(wsgi_app, BASE_PATH, bypass_paths={"/api/health"}))
app.wsgi_app = wsgi_app
@@ -1163,9 +1163,6 @@ def api_config() -> Response | tuple[Response, int]:
"show_combined_selector": app_config.get(
"SHOW_COMBINED_SELECTOR", True, user_id=db_user_id
),
"force_combined_search": app_config.get(
"FORCE_COMBINED_SEARCH", False, user_id=db_user_id
),
"books_output_mode": app_config.get("BOOKS_OUTPUT_MODE", "folder"),
"auto_open_downloads_sidebar": app_config.get("AUTO_OPEN_DOWNLOADS_SIDEBAR", True),
"hardcover_auto_remove_on_download": app_config.get(
+14 -17
View File
@@ -24,12 +24,12 @@ Dataclass representing a book from a metadata provider:
```python
@dataclass
class BookMetadata:
provider: str # Internal provider name (e.g., "hardcover")
provider_id: str # ID in that provider's system
provider: str # Internal provider name (e.g., "hardcover")
provider_id: str # ID in that provider's system
title: str
# Optional fields
provider_display_name: str # Human-readable name (e.g., "Hardcover")
provider_display_name: str # Human-readable name (e.g., "Hardcover")
authors: List[str]
isbn_10: str
isbn_13: str
@@ -39,7 +39,7 @@ class BookMetadata:
publish_year: int
language: str
genres: List[str]
source_url: str # Link to book on provider's site
source_url: str # Link to book on provider's site
display_fields: List[DisplayField] # Provider-specific display data
```
@@ -50,9 +50,9 @@ Provider-specific metadata for UI cards (ratings, page counts, reader counts, et
```python
@dataclass
class DisplayField:
label: str # e.g., "Rating", "Pages", "Readers"
value: str # e.g., "4.5", "496", "8,041"
icon: str # Icon name: "star", "book", "users", "editions"
label: str # e.g., "Rating", "Pages", "Readers"
value: str # e.g., "4.5", "496", "8,041"
icon: str # Icon name: "star", "book", "users", "editions"
```
### MetadataSearchOptions
@@ -64,7 +64,7 @@ Unified search options that work across all providers:
class MetadataSearchOptions:
query: str
search_type: SearchType = SearchType.GENERAL # GENERAL, TITLE, AUTHOR, ISBN
language: str = None # ISO 639-1 code (e.g., "en")
language: str = None # ISO 639-1 code (e.g., "en")
sort: SortOrder = SortOrder.RELEVANCE
limit: int = 40
page: int = 1
@@ -88,10 +88,10 @@ All providers must implement this interface:
```python
class MetadataProvider(ABC):
name: str # Internal identifier
display_name: str # Human-readable name
requires_auth: bool # True if API key required
supported_sorts: List[SortOrder] # Supported sort options
name: str # Internal identifier
display_name: str # Human-readable name
requires_auth: bool # True if API key required
supported_sorts: List[SortOrder] # Supported sort options
@abstractmethod
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
@@ -121,9 +121,9 @@ class MetadataProvider(ABC):
```python
from shelfmark.metadata_providers import register_provider
@register_provider("my_provider")
class MyProvider(MetadataProvider): ...
class MyProvider(MetadataProvider):
...
```
### Getting Providers
@@ -281,13 +281,11 @@ from shelfmark.config.env import (
METADATA_CACHE_BOOK_TTL,
)
@cacheable(ttl=METADATA_CACHE_SEARCH_TTL, key_prefix="myprovider:search")
def _search_cached(self, cache_key: str, options: MetadataSearchOptions):
# Cached search implementation
pass
@cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="myprovider:book")
def get_book(self, book_id: str):
# Cached book lookup
@@ -304,7 +302,6 @@ from shelfmark.metadata_providers.openlibrary import RateLimiter
# 90 requests per 60 seconds
rate_limiter = RateLimiter(max_requests=90, window_seconds=60)
def make_request(self):
rate_limiter.wait_if_needed() # Blocks if rate limited
# ... make request
-3
View File
@@ -709,6 +709,3 @@ with suppress(ImportError):
with suppress(ImportError):
from shelfmark.metadata_providers import googlebooks as googlebooks
with suppress(ImportError):
from shelfmark.metadata_providers import moly as moly
+4 -69
View File
@@ -1,7 +1,6 @@
"""Hardcover.app metadata provider. Requires API key."""
import re
import time
from contextlib import suppress
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -554,31 +553,6 @@ TITLE_SUGGESTION_FIELDS = "title,alternative_titles"
TITLE_SUGGESTION_WEIGHTS = "5,2"
TITLE_SUGGESTION_SORT = "_text_match:desc,users_count:desc"
# Hardcover forwards `sort` to Typesense's `sort_by` and rejects the whole search
# if it does not like the value -- an unknown field, a bare field name with no
# direction, more than three keys. A rejected search comes back as HTTP 200 with
# no GraphQL errors and a null `results` body, which is otherwise indistinguishable
# from "nothing matched". An empty sort is always accepted, so fall back to it and
# keep the fallback sticky for a while rather than paying for a doomed request on
# every search.
SORT_FALLBACK = ""
SORT_FALLBACK_TTL = 900.0
_sort_fallback_until = 0.0
def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
"""Report whether Hardcover answered a search with a null results body.
A search that genuinely matched nothing still returns a results object with
``found: 0``; only a rejected search nulls it out entirely.
"""
if not isinstance(result, dict):
return False
root = result.get("search", result)
if not isinstance(root, dict) or "results" not in root:
return False
return root["results"] is None
def _combine_headline_description(headline: str | None, description: str | None) -> str | None:
"""Combine headline (tagline) and description into a single description."""
@@ -1226,7 +1200,7 @@ class HardcoverProvider(MetadataProvider):
if not self.api_key or len(normalized_query) < HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH:
return []
result = self._execute_search_query(
result = self._execute_query(
SEARCH_FIELD_OPTIONS_QUERY,
{
"query": normalized_query,
@@ -1457,7 +1431,7 @@ class HardcoverProvider(MetadataProvider):
logger.debug("Invalid Hardcover series id field value: %s", normalized_value)
return None
result = self._execute_search_query(
result = self._execute_query(
SEARCH_FIELD_OPTIONS_QUERY,
{
"query": normalized_value,
@@ -2412,7 +2386,7 @@ class HardcoverProvider(MetadataProvider):
variables["weights"] = search_weights
try:
result = self._execute_search_query(graphql_query, variables)
result = self._execute_query(graphql_query, variables)
if not result:
logger.debug("Hardcover search: No result from API")
return SearchResult(books=[], page=options.page, total_found=0, has_more=False)
@@ -2654,7 +2628,7 @@ class HardcoverProvider(MetadataProvider):
raise RuntimeError(msg) from e
return None
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.UNAUTHORIZED:
if e.response.status_code == HTTPStatus.UNAUTHORIZED:
logger.exception("Hardcover API key is invalid")
if raise_on_error:
msg = "Hardcover API key is invalid"
@@ -2680,45 +2654,6 @@ class HardcoverProvider(MetadataProvider):
raise RuntimeError(msg) from e
return None
def _execute_search_query(self, query: str, variables: dict[str, Any]) -> dict | None:
"""Execute a search query, retrying without ``sort`` if Hardcover rejects it.
Returns None when the search was rejected, so callers report an empty
result rather than silently treating a failure as "nothing matched".
"""
global _sort_fallback_until
sort = variables.get("sort")
if sort and time.monotonic() < _sort_fallback_until:
variables = {**variables, "sort": SORT_FALLBACK}
sort = None
result = self._execute_query(query, variables)
if not _search_payload_rejected(result):
return result
if not sort:
logger.error(
"Hardcover rejected this search (query_type=%s, fields=%s) and returned "
"no result body",
variables.get("queryType", "Book"),
variables.get("fields"),
)
return None
logger.warning(
"Hardcover rejected sort '%s'; retrying searches without a sort order for %ss",
sort,
int(SORT_FALLBACK_TTL),
)
_sort_fallback_until = time.monotonic() + SORT_FALLBACK_TTL
retry = self._execute_query(query, {**variables, "sort": SORT_FALLBACK})
if _search_payload_rejected(retry):
logger.error("Hardcover rejected this search even without a sort order")
return None
return retry
def _parse_search_result(self, item: dict) -> BookMetadata | None:
"""Parse a search result item into BookMetadata."""
try:
-493
View File
@@ -1,493 +0,0 @@
"""Moly.hu metadata provider. Hungarian book catalog, no API key required.
Scraping approach (search URL, book-page structure, language mapping) adapted
from the Calibre Moly_hu plugin by Hoffer Csaba, Kloon, otapi, Dezso, Hokutya,
seeder and contributors (GPL v3, mobileread.com).
"""
import re
import threading
import time
import unicodedata
from collections import deque
from typing import Any, ClassVar
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup, Tag
from shelfmark.core.cache import cacheable
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
ActionButton,
CheckboxField,
HeadingField,
SettingsField,
register_settings,
)
from shelfmark.download.network import get_ssl_verify
from shelfmark.metadata_providers import (
BookMetadata,
DisplayField,
MetadataProvider,
MetadataSearchOptions,
SearchField,
SearchType,
SortOrder,
TextSearchField,
register_provider,
)
logger = setup_logger(__name__)
MOLY_BASE_URL = "https://moly.hu"
MOLY_BOOK_URL = f"{MOLY_BASE_URL}/konyvek/"
MOLY_SEARCH_URL = f"{MOLY_BASE_URL}/kereses?query="
# Be polite: moly.hu is a small community site
RATE_LIMIT_REQUESTS = 30
RATE_LIMIT_WINDOW_SECONDS = 60
REQUEST_HEADERS = {
"User-Agent": ("Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"),
"Accept-Language": "hu,en;q=0.7",
}
ISBN_13_LENGTH = 13
# Moly tags its foreign-language editions; everything else is Hungarian.
# Mapping from the Calibre Moly_hu plugin.
_LANGUAGE_TAG_MAP = {
"angol nyelvű": "en",
"n\xe9met nyelvű": "de",
"francia nyelvű": "fr",
"olasz nyelvű": "it",
"spanyol nyelvű": "es",
"orosz nyelvű": "ru",
"t\xf6r\xf6k nyelvű": "tr",
"g\xf6r\xf6g nyelvű": "el",
"k\xednai nyelvű": "zh",
"jap\xe1n nyelvű": "ja",
}
class RateLimiter:
"""Simple sliding window rate limiter."""
def __init__(self, max_requests: int, window_seconds: int) -> None:
"""Initialize rate limiter with max requests per time window."""
self.max_requests = max_requests
self.window_seconds = window_seconds
self.timestamps: deque[float] = deque()
self.lock = threading.Lock()
def wait_if_needed(self) -> None:
"""Block until a request is allowed (thread-safe)."""
wait_time = 0.0
with self.lock:
now = time.time()
cutoff = now - self.window_seconds
while self.timestamps and self.timestamps[0] < cutoff:
self.timestamps.popleft()
if len(self.timestamps) >= self.max_requests:
wait_time = self.timestamps[0] + self.window_seconds - now
if wait_time > 0:
logger.debug("Rate limited, waiting %0.2fs", wait_time)
time.sleep(wait_time)
with self.lock:
now = time.time()
cutoff = now - self.window_seconds
while self.timestamps and self.timestamps[0] < cutoff:
self.timestamps.popleft()
self.timestamps.append(time.time())
_rate_limiter = RateLimiter(RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_SECONDS)
def _clean_text(value: str | None) -> str | None:
"""Strip zero-width characters and collapse whitespace."""
if value is None:
return None
value = value.replace("​", "").replace("", "")
return " ".join(value.split())
def _normalize_for_match(value: str | None) -> str:
"""Accent-insensitive, punctuation-insensitive comparison form."""
if not value:
return ""
value = unicodedata.normalize("NFKD", value)
value = "".join(char for char in value if not unicodedata.combining(char))
value = "".join(char if char.isalnum() else " " for char in value)
return " ".join(value.lower().split())
def _absolute_url(url: str | None) -> str | None:
if not url:
return None
if url.startswith(("http://", "https://")):
return url
return MOLY_BASE_URL + url
def _valid_isbn(candidate: str) -> str | None:
"""Return a normalized ISBN-10/13 (digits, with optional X check digit), else None."""
digits = candidate.replace("-", "").strip()
if len(digits) == ISBN_13_LENGTH and digits.isdigit():
return digits
if len(digits) == 10 and re.fullmatch(r"\d{9}[\dXx]", digits):
return digits.upper()
return None
@register_provider("moly")
class MolyProvider(MetadataProvider):
"""Moly.hu metadata provider (HTML scraping, Hungarian catalog)."""
name = "moly"
display_name = "Moly.hu"
requires_auth = False
supported_sorts: ClassVar[tuple[SortOrder, ...]] = (SortOrder.RELEVANCE,)
search_fields: ClassVar[tuple[SearchField, ...]] = (
TextSearchField(
key="author",
label="Author",
description="Search by author name",
),
TextSearchField(
key="title",
label="Title",
description="Search by book title",
),
)
def __init__(self) -> None:
"""Initialize provider."""
self.session = requests.Session()
self.session.headers.update(REQUEST_HEADERS)
def is_available(self) -> bool:
"""Moly.hu needs no authentication."""
return True
def _fetch(self, url: str, timeout: int = 15) -> str | None:
_rate_limiter.wait_if_needed()
try:
response = self.session.get(url, timeout=timeout, verify=get_ssl_verify(MOLY_BASE_URL))
response.raise_for_status()
except requests.Timeout:
logger.warning("Moly.hu request timed out: %s", url)
return None
except requests.RequestException:
logger.exception("Moly.hu request failed: %s", url)
return None
return response.text
def search(self, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Search moly.hu's site search."""
if options.search_type == SearchType.ISBN:
result = self.search_by_isbn(options.query)
return [result] if result else []
# Moly's search is a single ranked page; no server-side pagination.
if options.page > 1:
return []
author_value = (options.fields.get("author") or "").strip()
title_value = (options.fields.get("title") or "").strip()
terms = " ".join(t for t in (author_value, title_value) if t)
query = terms or options.query.strip()
if not query:
return []
fields_key = ":".join(f"{k}={v}" for k, v in sorted(options.fields.items()))
cache_key = f"{query}:{options.search_type.value}:{options.limit}:{fields_key}"
return self._search_cached(cache_key, query, options.limit) or []
@cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="moly:search")
def _search_cached(self, cache_key: str, query: str, limit: int) -> list[BookMetadata] | None:
# Return None (not []) on fetch failure so the failure is not cached.
html = self._fetch(MOLY_SEARCH_URL + quote(query.encode("utf-8")))
if html is None:
return None
soup = BeautifulSoup(html, "html.parser")
books: list[BookMetadata] = []
seen: set[str] = set()
for anchor in soup.select("#content div.search_area a.book_selector"):
href = anchor.get("href") or ""
match = re.search(r"/konyvek/([^/?#]+)", str(href))
if not match:
continue
slug = match.group(1)
if slug in seen:
continue
# No separator: moly wraps matched search terms in <strong> even
# mid-word ("Lis<strong>a</strong> Jewell"), so inserting one
# would split words at highlight boundaries.
text = _clean_text(anchor.get_text()) or ""
author, _, title = text.partition(":")
if not title:
# Result rows are "Author: Title"; skip anything else.
continue
author = author.strip()
title = title.strip()
seen.add(slug)
books.append(
BookMetadata(
provider=self.name,
provider_id=slug,
provider_display_name=self.display_name,
title=title,
authors=[author] if author else [],
cover_url=self._cover_for_result(soup, text),
source_url=MOLY_BOOK_URL + slug,
language="hu",
search_title=title,
search_author=author or None,
display_fields=self._result_display_fields(anchor),
)
)
if len(books) >= limit:
break
logger.info("Moly.hu search '%s' returned %s results", query, len(books))
return books
def _cover_for_result(self, soup: BeautifulSoup, result_text: str) -> str | None:
"""Find the search-result thumbnail whose alt matches 'Author: Title'."""
target = _normalize_for_match(result_text)
if not target:
return None
for img in soup.select("#content img.tooltip[alt]"):
if _normalize_for_match(str(img.get("alt") or "")) == target:
return _absolute_url(str(img.get("src") or "")) or None
return None
def _result_display_fields(self, anchor: Tag) -> list[DisplayField]:
fields: list[DisplayField] = []
parent = anchor.parent
if parent is None:
return fields
like = parent.select_one("span.like_count")
if like:
fields.append(
DisplayField(label="Rating", value=like.get_text(strip=True), icon="star")
)
series = parent.select_one('a[href*="/sorozatok/"]')
if series:
fields.append(
DisplayField(
label="Series",
value=series.get_text(strip=True).strip("()"),
icon="editions",
)
)
return fields
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="moly:book")
def get_book(self, book_id: str) -> BookMetadata | None:
"""Get book details by moly.hu slug (e.g. 'mocsidzuki-mai-a-telihold-kavezo')."""
html = self._fetch(MOLY_BOOK_URL + quote(book_id))
if html is None:
return None
soup = BeautifulSoup(html, "html.parser")
title = self._parse_title(soup)
authors = [_clean_text(a.get_text()) or "" for a in soup.select("#content div.authors a")]
authors = [a for a in authors if a]
if not title or not authors:
logger.warning("Moly.hu book page missing title/authors: %s", book_id)
return None
isbn_13, isbn_10 = self._parse_isbns(soup)
series = self._parse_series(soup)
tags = [_clean_text(t.get_text()) or "" for t in soup.select("#book_tags a.tag")]
tags = [t for t in tags if t]
display_fields: list[DisplayField] = []
rating = soup.select_one("#content .rating .like_count")
if rating:
display_fields.append(
DisplayField(label="Rating", value=rating.get_text(strip=True), icon="star")
)
if series:
display_fields.append(DisplayField(label="Series", value=series, icon="editions"))
return BookMetadata(
provider=self.name,
provider_id=book_id,
provider_display_name=self.display_name,
title=title,
authors=authors,
isbn_13=isbn_13,
isbn_10=isbn_10,
cover_url=self._parse_cover(soup),
description=self._parse_description(soup),
publisher=self._parse_publisher(soup),
publish_year=self._parse_publish_year(soup),
language=self._parse_language(tags),
genres=tags,
source_url=MOLY_BOOK_URL + book_id,
search_title=title,
search_author=authors[0],
display_fields=display_fields,
)
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="moly:isbn")
def search_by_isbn(self, isbn: str) -> BookMetadata | None:
"""Moly's site search resolves ISBN queries directly."""
isbn = isbn.replace("-", "").strip()
if not isbn:
return None
html = self._fetch(MOLY_SEARCH_URL + quote(isbn))
if html is None:
return None
soup = BeautifulSoup(html, "html.parser")
anchor = soup.select_one("#content div.search_area a.book_selector[href]")
if not anchor:
return None
match = re.search(r"/konyvek/([^/?#]+)", str(anchor.get("href")))
if not match:
return None
return self.get_book(match.group(1))
def _parse_title(self, soup: BeautifulSoup) -> str | None:
node = soup.select_one("#content .head_title h1 span.item")
if node:
# The series link is nested inside this span; only direct text
# belongs to the book title.
direct = "".join(node.find_all(string=True, recursive=False))
title = _clean_text(direct)
if title:
return title
node = soup.select_one("#content .book > span")
if node:
return _clean_text(node.get_text())
return None
def _parse_series(self, soup: BeautifulSoup) -> str | None:
node = soup.select_one('#content h1 a[href*="/sorozatok/"]')
if not node:
return None
return (_clean_text(node.get_text()) or "").strip("()") or None
def _parse_isbns(self, soup: BeautifulSoup) -> tuple[str | None, str | None]:
isbn_13 = isbn_10 = None
editions = soup.select("#content .items .edition") or soup.select("#content .items > div")
for edition in editions:
text = edition.get_text(" ")
for candidate in re.findall(r"(?<!\d)[\d-]{10,17}(?!\d)", text):
isbn = _valid_isbn(candidate)
if not isbn:
continue
if len(isbn) == ISBN_13_LENGTH and not isbn_13:
isbn_13 = isbn
elif len(isbn) != ISBN_13_LENGTH and not isbn_10:
isbn_10 = isbn
if isbn_13:
break
return isbn_13, isbn_10
def _parse_cover(self, soup: BeautifulSoup) -> str | None:
node = soup.select_one("#content .coverbox a.zoom[href]")
if node:
return _absolute_url(str(node.get("href")))
img = soup.select_one("#content .coverbox img[src]")
if img:
return _absolute_url(str(img.get("src")))
return None
def _parse_description(self, soup: BeautifulSoup) -> str | None:
node = soup.select_one("#content #full_description")
if node is None:
node = soup.select_one("#content div.text")
if node is None:
return None
spoiler_warning = "Vigyázat! Cselekményleírást tartalmaz."
parts = []
for text in node.stripped_strings:
cleaned = _clean_text(text) or ""
if cleaned.startswith(spoiler_warning):
cleaned = cleaned[len(spoiler_warning) :].strip()
if cleaned:
parts.append(cleaned)
return "\n".join(parts) or None
def _parse_publisher(self, soup: BeautifulSoup) -> str | None:
node = soup.select_one('#content .items .edition a[href*="/kiadok/"]')
if node:
return _clean_text(node.get_text())
return None
def _parse_publish_year(self, soup: BeautifulSoup) -> int | None:
editions = soup.select("#content .items .edition") or soup.select("#content .items > div")
for edition in editions:
match = re.search(r"\b(\d{4})\b", edition.get_text(" "))
if match:
return int(match.group(1))
return None
def _parse_language(self, tags: list[str]) -> str:
for tag in tags:
code = _LANGUAGE_TAG_MAP.get(tag.lower().strip())
if code:
return code
return "hu"
def _test_moly_connection() -> dict[str, Any]:
"""Test connectivity to moly.hu."""
try:
provider = MolyProvider()
response = provider.session.get(
MOLY_SEARCH_URL + quote("teszt"),
timeout=10,
verify=get_ssl_verify(MOLY_BASE_URL),
)
response.raise_for_status()
except requests.Timeout:
return {"success": False, "message": "Connection timed out"}
except requests.RequestException as e:
return {"success": False, "message": f"Connection failed: {e}"}
if "moly" in response.text.lower():
return {"success": True, "message": "Successfully connected to moly.hu"}
return {"success": False, "message": "Unexpected response from moly.hu"}
@register_settings("moly", "Moly.hu", icon="library", order=54, group="metadata_providers")
def moly_settings() -> list[SettingsField]:
"""Moly.hu metadata provider settings."""
return [
HeadingField(
key="moly_heading",
title="Moly.hu",
description=(
"Hungarian community book catalog with excellent coverage of "
"Hungarian editions and translations. No API key required."
),
link_url="https://moly.hu",
link_text="moly.hu",
),
CheckboxField(
key="MOLY_ENABLED",
label="Enable Moly.hu",
description="Enable Moly.hu as a metadata provider for book searches",
default=False,
),
ActionButton(
key="test_connection",
label="Test Connection",
description="Verify moly.hu is accessible",
style="primary",
callback=_test_moly_connection,
),
]
+3 -3
View File
@@ -214,7 +214,7 @@ class OpenLibraryProvider(MetadataProvider):
logger.warning("Open Library search timed out")
return []
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
if e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
logger.warning("Open Library service unavailable (503)")
else:
logger.exception("Open Library HTTP error")
@@ -253,7 +253,7 @@ class OpenLibraryProvider(MetadataProvider):
logger.warning("Open Library get_book timed out")
return None
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.NOT_FOUND:
if e.response.status_code == HTTPStatus.NOT_FOUND:
logger.debug("Open Library work not found: %s", book_id)
else:
logger.exception("Open Library HTTP error")
@@ -314,7 +314,7 @@ class OpenLibraryProvider(MetadataProvider):
return self._parse_edition(edition, clean_isbn)
except requests.HTTPError as e:
if e.response is not None and e.response.status_code == HTTPStatus.NOT_FOUND:
if e.response.status_code == HTTPStatus.NOT_FOUND:
logger.debug("Open Library ISBN not found: %s", isbn)
else:
logger.exception("Open Library ISBN search HTTP error")
+1 -7
View File
@@ -163,7 +163,6 @@ class SortOption:
label: str # Display label in the sort dropdown
sort_key: str # Field to sort by on the Release object
default_direction: Literal["asc", "desc"] = "desc" # Which way "best first" runs
@dataclass
@@ -262,12 +261,7 @@ def serialize_column_config(config: ReleaseColumnConfig) -> dict[str, Any]:
# Include extra sort options (sort entries not tied to a column)
if config.extra_sort_options:
result["extra_sort_options"] = [
{
"label": opt.label,
"sort_key": opt.sort_key,
"default_direction": opt.default_direction,
}
for opt in config.extra_sort_options
{"label": opt.label, "sort_key": opt.sort_key} for opt in config.extra_sort_options
]
# Include action button if specified (replaces default expand search)
@@ -417,18 +417,6 @@ def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") ->
# Clean up info hash (remove whitespace, ensure uppercase)
info_hash = re.sub(r"\s+", "", info_hash).upper()
# Validate: SHA1 = 40 hex chars, SHA256 = 64 hex chars
if not re.match(r"^[0-9A-F]{40}$|^[0-9A-F]{64}$", info_hash):
logger.warning("Info Hash invalid (got %r), trying magnet fallback.", info_hash)
# Fallback: search entire page for a complete magnet link (e.g. posted in comments)
magnet_match = re.search(r"magnet:\?xt=urn:btih:([0-9a-fA-F]{40,64})", detail_html)
if magnet_match:
info_hash = magnet_match.group(1).upper()
logger.info("Found hash via magnet fallback: %s", info_hash)
else:
logger.warning("No valid magnet link found on page, giving up.")
return None
# 2. Extract Trackers
# Find all <td> containing udp:// or http://
trackers = []
@@ -9,7 +9,6 @@ if TYPE_CHECKING:
from shelfmark.metadata_providers import BookMetadata
from shelfmark.core.config import config
from shelfmark.core.languages import normalize_language
from shelfmark.core.logger import setup_logger
from shelfmark.release_sources import (
ColumnAlign,
@@ -44,6 +43,41 @@ def _coerce_positive_int(value: object, default: int) -> int:
# Map language names to ISO 639-1 codes (matching frontend color maps)
LANGUAGE_MAP = {
"english": "en",
"spanish": "es",
"french": "fr",
"german": "de",
"italian": "it",
"portuguese": "pt",
"russian": "ru",
"japanese": "ja",
"chinese": "zh",
"dutch": "nl",
"swedish": "sv",
"norwegian": "no",
"danish": "da",
"finnish": "fi",
"polish": "pl",
"czech": "cs",
"hungarian": "hu",
"korean": "ko",
"arabic": "ar",
"hebrew": "he",
"turkish": "tr",
"greek": "el",
"hindi": "hi",
"thai": "th",
"vietnamese": "vi",
"indonesian": "id",
"ukrainian": "uk",
"romanian": "ro",
"bulgarian": "bg",
"catalan": "ca",
"croatian": "hr",
"slovenian": "sl",
"serbian": "sr",
}
def _split_title_and_author(raw_title: str) -> tuple[str, str | None]:
@@ -85,9 +119,8 @@ def _map_language(language: str) -> str | None:
if not language:
return None
# Fall back to the raw value so an unrecognised language is still shown
# rather than silently dropped from the release row.
return normalize_language(language) or language.lower().strip()
lang_lower = language.lower().strip()
return LANGUAGE_MAP.get(lang_lower, lang_lower)
def _parse_bitrate_to_kbps(bitrate: str | None) -> int | None:
@@ -205,8 +238,8 @@ class AudiobookBaySource(ReleaseSource):
exact_phrase=exact_phrase,
)
# Fallback to broad matching if exact phrase returns nothing (manual or auto query).
if exact_phrase and not results:
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
if exact_phrase and not results and not plan.manual_query:
logger.info(
"No exact phrase results, retrying AudiobookBay search without quotes"
)
@@ -255,7 +288,7 @@ class AudiobookBaySource(ReleaseSource):
size_str = result.get("size")
size_bytes = parse_size(size_str) if size_str else None
language_raw = result.get("language")
language_code = _map_language(language_raw) if language_raw else "en"
language_code = _map_language(language_raw) if language_raw else None
bitrate = result.get("bitrate")
bitrate_kbps = _parse_bitrate_to_kbps(bitrate)
+23 -293
View File
@@ -3,12 +3,9 @@
import itertools
import json
import re
import threading
import time
import unicodedata
from dataclasses import replace
from http import HTTPStatus
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, NoReturn, TypedDict
from urllib.parse import quote, urlparse
@@ -18,7 +15,6 @@ from bs4.element import NavigableString
from shelfmark.config.env import DEBUG_SKIP_SOURCES, TMP_DIR
from shelfmark.core.config import config
from shelfmark.core.languages import language_alias_map
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask, SearchFilters, build_filename
from shelfmark.core.utils import CONTENT_TYPES, get_aa_content_type_dir
@@ -201,49 +197,6 @@ _SOURCE_FAILURE_THRESHOLD = 4
_MIN_VALID_FILE_SIZE = 10 * 1024
_AA_COUNTDOWN_MAX_SECONDS = 300
# --- Distant-path language detection ---
_DISTANT_PATH_EXTENSIONS = (
"epub",
"mobi",
"azw3",
"fb2",
"djvu",
"cbz",
"cbr",
"pdf",
"zip",
"rar",
"m4b",
"mp3",
)
_DISTANT_PATH_EXTENSION_PATTERN = "|".join(re.escape(e) for e in _DISTANT_PATH_EXTENSIONS)
_DISTANT_PATH_PATTERN = re.compile(
rf"(?:[A-Za-z0-9._-]+/)?[A-Za-z]:(?:\\|/)[^\n\r<>\"]+?\.(?:{_DISTANT_PATH_EXTENSION_PATTERN})\b",
re.IGNORECASE,
)
_DISTANT_PATH_FALLBACK_PATTERN = re.compile(
r"(?:[A-Za-z0-9._-]+/)?[A-Za-z]:(?:\\|/)[^\n\r<>\"]+",
re.IGNORECASE,
)
_BRACKETED_LANGUAGE_CODE_PATTERN = re.compile(
r"\[(?:bd[\s._-]*)?([A-Za-z]{2,3})\]",
re.IGNORECASE,
)
_KEYED_LANGUAGE_CODE_PATTERN = re.compile(
r"\b(?:bd|lang(?:uage)?)\s*[:._-]?\s*([A-Za-z]{2,3})\b",
re.IGNORECASE,
)
_LANGUAGE_CODE_TOKEN_PATTERN = re.compile(
r"(?:^|[\s_./\\\-\[(])([A-Za-z]{2,3})(?=$|[\s_./\\\-)\]])"
)
_LANGUAGE_NAME_TOKEN_PATTERN = re.compile(r"[a-z]{4,}(?:-[a-z0-9]+)?")
_LANGUAGE_ALIAS_TO_CODE: dict[str, str] | None = None
_LANGUAGE_ALIAS_LOCK = threading.Lock()
_LANGUAGE_PLACEHOLDERS = frozenset({"", "-", "--", "unknown", "unk", "n/a", "na"})
# Short codes that appear in common words — require bracket/key context to accept
_AMBIGUOUS_SHORT_LANGUAGE_CODES = frozenset({"de", "en", "it", "la", "no", "or", "is", "in"})
# Sources that require Cloudflare bypass
_CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "welib"})
@@ -251,161 +204,6 @@ _CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "weli
_AA_PAGE_SOURCES = frozenset({"aa-slow-nowait", "aa-slow-wait"})
def _is_language_from_path_enabled() -> bool:
return bool(config.get("DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH", False))
def _normalize_language_token(value: str) -> str:
normalized = value.strip().lower()
for dash in ("‑", "–", "—", "−"):
normalized = normalized.replace(dash, "-")
return normalized
def _fold_text(value: str) -> str:
normalized = unicodedata.normalize("NFKD", value)
return "".join(c for c in normalized if not unicodedata.combining(c)).lower()
def _language_alias_to_code() -> dict[str, str]:
"""Alias to code map, delegating to the shared language data."""
global _LANGUAGE_ALIAS_TO_CODE
cached = _LANGUAGE_ALIAS_TO_CODE
if cached is not None:
return cached
with _LANGUAGE_ALIAS_LOCK:
cached = _LANGUAGE_ALIAS_TO_CODE
if cached is not None:
return cached
_LANGUAGE_ALIAS_TO_CODE = language_alias_map()
return _LANGUAGE_ALIAS_TO_CODE
def _extract_distant_path(row: Tag, *, enabled: bool) -> str | None:
"""Extract the Windows-style file path from an AA search result row."""
if not enabled:
return None
def _normalize_candidate(text: str) -> str:
normalized = re.sub(r"\s*([\\/])\s*", r"\1", text)
normalized = re.sub(r":\s*([\\/])", r":\1", normalized)
return re.sub(
r"\s+\.(epub|mobi|azw3|fb2|djvu|cbz|cbr|pdf|zip|rar|m4b|mp3)\b",
r".\1",
normalized,
flags=re.IGNORECASE,
)
candidates = [row.get_text(" ", strip=True)]
for cell in row.find_all("td"):
cell_text = cell.get_text(" ", strip=True)
if cell_text:
candidates.append(cell_text)
best: str | None = None
for text in candidates:
for match in _DISTANT_PATH_PATTERN.findall(_normalize_candidate(text)):
candidate = match.strip().rstrip(".,;")
if best is None or len(candidate) > len(best):
best = candidate
if best is not None:
return best
for text in candidates:
for match in _DISTANT_PATH_FALLBACK_PATTERN.findall(_normalize_candidate(text)):
candidate = match.strip().rstrip(".,;")
if best is None or len(candidate) > len(best):
best = candidate
return best
def _detect_language_from_distant_path(path: str | None) -> str | None:
"""Infer a language code from distant-path tags such as [BD FR] or [Fr]."""
if not path:
return None
aliases = _language_alias_to_code()
if not aliases:
return None
folded_path = _fold_text(path)
strong_candidates: list[str] = []
for code in _BRACKETED_LANGUAGE_CODE_PATTERN.findall(path):
normalized = _normalize_language_token(code)
if normalized in aliases:
strong_candidates.append(aliases[normalized])
for code in _KEYED_LANGUAGE_CODE_PATTERN.findall(path):
normalized = _normalize_language_token(code)
if normalized in aliases:
strong_candidates.append(aliases[normalized])
non_ambiguous = [c for c in strong_candidates if c not in _AMBIGUOUS_SHORT_LANGUAGE_CODES]
if non_ambiguous:
return non_ambiguous[0]
for token in _LANGUAGE_NAME_TOKEN_PATTERN.findall(folded_path):
normalized = _normalize_language_token(token)
if normalized in aliases:
candidate = aliases[normalized]
if candidate not in _AMBIGUOUS_SHORT_LANGUAGE_CODES:
return candidate
if strong_candidates:
return strong_candidates[0]
for code in _LANGUAGE_CODE_TOKEN_PATTERN.findall(path):
normalized = _normalize_language_token(code)
if normalized in _AMBIGUOUS_SHORT_LANGUAGE_CODES:
continue
if normalized in aliases:
return aliases[normalized]
return None
def _is_missing_or_placeholder_language(language: str | None) -> bool:
if language is None:
return True
return _normalize_language_token(language) in _LANGUAGE_PLACEHOLDERS
def _normalize_requested_languages(languages: list[str] | None) -> set[str]:
if not languages:
return set()
aliases = _language_alias_to_code()
normalized: set[str] = set()
for value in languages:
token = _normalize_language_token(str(value))
if not token or token == "all": # noqa: S105 - "all" is a language sentinel
continue
normalized.add(aliases.get(token, token))
return normalized
def _book_matches_requested_languages(book_language: str | None, requested: set[str]) -> bool:
"""Return True when a book's language matches the requested filter.
Books with unknown/missing language always pass — the server-side &lang= filter
already narrowed the result set, so dropping unlabelled rows hides valid results.
"""
if not requested:
return True
if not book_language:
return True
aliases = _language_alias_to_code()
normalized_book = aliases.get(
_normalize_language_token(book_language),
_normalize_language_token(book_language),
)
return normalized_book in requested
def _is_configured_zlib_link(url: str) -> bool:
"""Return True when a URL belongs to a configured Z-Library mirror."""
from shelfmark.core.mirrors import get_zlib_cookie_domains
@@ -562,17 +360,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
filters_query = ""
path_language_enabled = _is_language_from_path_enabled()
requested_langs = _normalize_requested_languages(filters.lang)
# When path-language inference is on and a language is requested, skip the
# server-side &lang= filter: lgli files often have no AA language metadata
# and would be excluded before we can infer language from the distant path.
# Local filtering below handles the narrowing instead.
if not (path_language_enabled and requested_langs):
for value in filters.lang or []:
if value and value != "all":
filters_query += f"&lang={quote(value)}"
for value in filters.lang or []:
if value and value != "all":
filters_query += f"&lang={quote(value)}"
if filters.sort and filters.sort != "relevance":
filters_query += f"&sort={quote(filters.sort)}"
@@ -607,13 +397,14 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
raise SearchUnavailableError(msg)
if "No files found." in html:
logger.info("No books found for query: %s", query)
return []
soup = BeautifulSoup(_html_response_text(html), "html.parser")
tbody = soup.find("table")
if tbody is None:
if "No files found." in html:
logger.info("No books found for query: %s", query)
return []
logger.warning("No results table found for query: %s", query)
msg = "No books found. Please try another query."
raise RuntimeError(msg)
@@ -627,9 +418,6 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
if book:
books.append(book)
if path_language_enabled and requested_langs:
books = [b for b in books if _book_matches_requested_languages(b.language, requested_langs)]
supported_formats = _get_supported_formats()
books.sort(
@@ -683,23 +471,10 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
if not record_id:
return None
path_language_enabled = _is_language_from_path_enabled()
distant_path = _extract_distant_path(row, enabled=path_language_enabled)
preview_img = cells[0].find("img")
preview = _get_attr(preview_img, "src") if isinstance(preview_img, Tag) else None
title_span = cells[1].find("span")
if isinstance(title_span, Tag):
# AA nests related-edition spans inside the main title span — take only direct text.
direct = " ".join(
str(c).strip()
for c in title_span.children
if isinstance(c, NavigableString) and str(c).strip()
).strip()
title = direct or _first_stripped_text(title_span)
else:
title = None
title = _first_stripped_text(cells[1].find("span"))
author = _first_stripped_text(cells[2].find("span"))
publisher = _first_stripped_text(cells[3].find("span"))
year = _first_stripped_text(cells[4].find("span"))
@@ -708,19 +483,18 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
file_format = _first_stripped_text(cells[9].find("span"))
size = _first_stripped_text(cells[10].find("span"))
# Only title and format are truly required — lgli rows often have sparse metadata
if title is None or file_format is None:
if (
title is None
or author is None
or publisher is None
or year is None
or language is None
or content is None
or file_format is None
or size is None
):
return None
# Skip entries where the title is a catalog format descriptor, not a real title
# e.g. "Book/Online Audio", "Print book" — lgli metadata pollution
if title and "/" in title and len(title) < 40 and not any(c.isdigit() for c in title):
return None
if path_language_enabled and _is_missing_or_placeholder_language(language):
detected = _detect_language_from_distant_path(distant_path)
language = detected or "unknown"
return BrowseRecord(
id=record_id,
title=title,
@@ -733,7 +507,6 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
content=content.lower() if content else None,
format=file_format.lower() if file_format else None,
size=size,
download_path=distant_path,
)
except (AttributeError, IndexError, KeyError, TypeError) as e:
logger.error_trace(f"Error parsing search result row: {e}")
@@ -1456,9 +1229,6 @@ def _get_download_url(
return downloader.get_absolute_url(link, url)
_AA_COUNTDOWN_MAX_RETRIES = 3
def _extract_slow_download_url(
soup: BeautifulSoup,
link: str,
@@ -1467,7 +1237,6 @@ def _extract_slow_download_url(
status_callback: Callable[[str, str | None], None] | None,
selector: network.AAMirrorSelector,
source_context: str | None = None,
_countdown_attempts: int = 0,
) -> str:
"""Extract download URL from AA slow download pages."""
html_str = str(soup)
@@ -1532,14 +1301,6 @@ def _extract_slow_download_url(
countdown_seconds = _extract_countdown_seconds(soup, html_str)
if countdown_seconds > 0:
if _countdown_attempts >= _AA_COUNTDOWN_MAX_RETRIES:
logger.warning(
"Countdown retry limit (%s) reached for %s, giving up",
_AA_COUNTDOWN_MAX_RETRIES,
title,
)
return ""
max_countdown_seconds = 600
sleep_time = min(countdown_seconds, max_countdown_seconds)
if countdown_seconds > max_countdown_seconds:
@@ -1548,13 +1309,7 @@ def _extract_slow_download_url(
countdown_seconds,
max_countdown_seconds,
)
logger.info(
"AA waitlist: %ss for %s (attempt %s/%s)",
sleep_time,
title,
_countdown_attempts + 1,
_AA_COUNTDOWN_MAX_RETRIES,
)
logger.info("AA waitlist: %ss for %s", sleep_time, title)
# Live countdown with status updates
for remaining in range(sleep_time, 0, -1):
@@ -1575,31 +1330,12 @@ def _extract_slow_download_url(
if status_callback and source_context:
status_callback("resolving", f"{source_context} - Fetching")
html = downloader.html_get_page(
link, selector=selector, cancel_flag=cancel_flag, status_callback=status_callback
)
if not html:
return ""
new_soup = BeautifulSoup(_html_response_text(html), "html.parser")
return _extract_slow_download_url(
new_soup,
link,
title,
cancel_flag,
status_callback,
selector,
source_context,
_countdown_attempts + 1,
return _get_download_url(
link, title, cancel_flag, status_callback, selector, source_context
)
link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
logger.warning("No download URL found. First 10 links: %s", link_texts)
# A bypassed page with no AA download links often means the network served a wrong
# page (e.g. an ISP block page) instead of Anna's Archive. Probe for DNS interference
# so we can give the user an actionable hint instead of a generic failure.
host = urlparse(link).hostname or ""
if host:
network.note_possible_dns_interference(host)
return ""
@@ -1910,6 +1646,7 @@ class DirectDownloadSource(ReleaseSource):
except Exception:
logger.exception("Search error")
logger.info("Found %s releases via title+author", len(all_results))
return [_browse_record_to_release(record) for record in all_results]
def is_available(self) -> bool:
@@ -2032,14 +1769,7 @@ class DirectDownloadHandler(DownloadHandler):
return None
if not success_url:
if network.dns_interference_detected():
status_callback(
"error",
"All sources failed - your network/ISP appears to be blocking "
"Anna's Archive. Enable DNS-over-HTTPS in settings.",
)
else:
status_callback("error", "All download sources failed")
status_callback("error", "All download sources failed")
return None
# Return temp path - orchestrator handles post-processing (archive extraction, ingest)
+46 -21
View File
@@ -13,6 +13,7 @@ from typing import Any
from shelfmark.config import env
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.release_sources import Release, ReleaseProtocol
logger = setup_logger(__name__)
@@ -55,6 +56,12 @@ def _coerce_timestamp(value: object) -> float:
return 0.0
def _generate_cache_key(provider: str, provider_id: str, content_type: str | None = None) -> str:
"""Generate a cache key from provider, provider_id, and content type."""
normalized_content_type = "audiobook" if check_audiobook(content_type) else "ebook"
return f"{provider}:{provider_id}:{normalized_content_type}"
def _load_cache() -> dict[str, Any]:
"""Load cache from disk."""
try:
@@ -96,17 +103,17 @@ def _dict_to_release(data: dict[str, Any]) -> Release:
def get_cached_results(
cache_key: str,
provider: str,
provider_id: str,
content_type: str | None = None,
ttl_seconds: int | None = None,
) -> dict[str, Any] | None:
"""Get the cached IRC answer for a query identity (server:channel:query).
The cache stores the whole answer (releases for all content types) under the query
identity, so it is not isolated by book or content type. Callers filter by content
type after reading.
"""Get cached search results for a book.
Args:
cache_key: Query identity (e.g. "server:channel:query")
provider: Metadata provider name (e.g., "hardcover", "openlibrary")
provider_id: Book ID in the provider's system
content_type: Search content type for cache isolation
ttl_seconds: Cache TTL in seconds (from settings)
Returns:
@@ -120,6 +127,8 @@ def get_cached_results(
ttl_value = config.get("IRC_CACHE_TTL", DEFAULT_CACHE_TTL)
ttl_seconds = _coerce_cache_ttl(ttl_value, DEFAULT_CACHE_TTL)
cache_key = _generate_cache_key(provider, provider_id, content_type)
with _cache_lock:
cache = _load_cache()
entry = cache.get("entries", {}).get(cache_key)
@@ -132,9 +141,10 @@ def get_cached_results(
age = time.time() - cached_at
if ttl_seconds != 0 and age > ttl_seconds:
title = entry.get("title", cache_key)
logger.debug(
"IRC cache expired for '%s' (age: %.0fs > TTL: %ss)",
entry.get("title", cache_key),
title,
age,
ttl_seconds,
)
@@ -143,36 +153,44 @@ def get_cached_results(
# Convert dicts back to Release objects
releases = [_dict_to_release(r) for r in entry.get("releases", [])]
online_servers = entry.get("online_servers", [])
title = entry.get("title", "")
logger.info(
"IRC cache hit for '%s' (%s releases, age: %.0fs)",
entry.get("title", ""),
title,
len(releases),
age,
)
return {
"releases": releases,
"online_servers": entry.get("online_servers", []),
"online_servers": online_servers,
"cached_at": cached_at,
}
def cache_results(
cache_key: str,
provider: str,
provider_id: str,
title: str,
releases: list[Release],
content_type: str | None = None,
online_servers: list[str] | None = None,
) -> None:
"""Cache the whole IRC answer for a query identity.
"""Cache search results for a book.
Args:
cache_key: Query identity (e.g. "server:channel:query")
title: Query text (for logging/display)
releases: All Release objects from the search (every content type)
provider: Metadata provider name
provider_id: Book ID in the provider's system
title: Book title (for logging/display)
releases: List of Release objects from search
content_type: Search content type for cache isolation
online_servers: List of online server nicks (optional)
"""
cache_key = _generate_cache_key(provider, provider_id, content_type)
with _cache_lock:
cache = _load_cache()
@@ -180,6 +198,9 @@ def cache_results(
cache["entries"] = {}
cache["entries"][cache_key] = {
"provider": provider,
"provider_id": provider_id,
"content_type": "audiobook" if check_audiobook(content_type) else "ebook",
"title": title,
"releases": [_release_to_dict(r) for r in releases],
"online_servers": list(online_servers) if online_servers else [],
@@ -190,23 +211,27 @@ def cache_results(
logger.info("Cached %s IRC releases for '%s'", len(releases), title)
def invalidate_cache(cache_key: str) -> bool:
def invalidate_cache(provider: str, provider_id: str, content_type: str | None = None) -> bool:
"""Remove a specific entry from the cache.
Args:
cache_key: Query identity to remove
provider: Metadata provider name
provider_id: Book ID in the provider's system
content_type: Search content type for cache isolation
Returns:
True if entry was found and removed
"""
cache_key = _generate_cache_key(provider, provider_id, content_type)
with _cache_lock:
cache = _load_cache()
entries = cache.get("entries", {})
entry = cache.get("entries", {}).get(cache_key)
title = entry.get("title", cache_key) if entry else cache_key
if cache_key in entries:
title = entries[cache_key].get("title", cache_key)
del entries[cache_key]
if cache_key in cache.get("entries", {}):
del cache["entries"][cache_key]
_save_cache(cache)
logger.info("Invalidated IRC cache for '%s'", title)
return True
+32 -68
View File
@@ -25,8 +25,6 @@ logger = setup_logger(__name__)
# Timing
SOCKET_TIMEOUT = 300.0 # 5 minutes - long because we wait for DCC offers
RECV_BUFFER = 4096
# How often a deadline-bound read wakes up to re-check the clock
POLL_INTERVAL = 2.0
# IRC channel user prefixes that indicate elevated status (ops, voice, etc.)
# These are the download bots/servers
@@ -250,22 +248,11 @@ class IRCClient:
# 366 = RPL_ENDOFNAMES - channel join is complete
if msg.command == "366":
if not self.online_servers:
# Joining a channel that doesn't exist on this network
# silently creates an empty one, so an empty name list is
# the only hint that the channel name is wrong.
logger.warning(
"Joined #%s but no servers are online - the channel may "
"be empty or not exist on %s",
channel,
self.server,
)
else:
logger.info(
"Joined #%s - %s servers online",
channel,
len(self.online_servers),
)
logger.info(
"Joined #%s - %s servers online",
channel,
len(self.online_servers),
)
return
# Check for errors (e.g., banned, channel doesn't exist)
@@ -309,47 +296,27 @@ class IRCClient:
data = f"{message}\r\n".encode()
self._socket.sendall(data)
def _recv_lines(self, deadline: float | None = None) -> Iterator[str]:
"""Receive and yield complete CRLF-delimited IRC lines.
A deadline stops the read once it passes, even if nothing ever arrives.
Callers time out by watching the messages they receive, so on a channel
with no traffic at all there is nothing to watch: the recv would just
keep blocking for SOCKET_TIMEOUT and retrying forever.
"""
def _recv_lines(self) -> Iterator[str]:
"""Receive and yield complete CRLF-delimited IRC lines."""
sock = self._require_socket()
original_timeout = sock.gettimeout()
while True:
# Check if we have a complete line in buffer
while "\r\n" in self._buffer:
line, self._buffer = self._buffer.split("\r\n", 1)
if line:
yield line
try:
while True:
# Check if we have a complete line in buffer
while "\r\n" in self._buffer:
line, self._buffer = self._buffer.split("\r\n", 1)
if line:
yield line
if deadline is not None:
remaining = deadline - time.time()
if remaining <= 0:
return
# Wake up often enough to notice the deadline pass
sock.settimeout(min(remaining, POLL_INTERVAL))
# Read more data
try:
data = sock.recv(RECV_BUFFER)
if not data:
return # Connection closed
self._buffer += data.decode("utf-8", errors="replace")
except TimeoutError:
continue # Keep waiting (the deadline is re-checked above)
except OSError as e:
logger.warning("Socket error: %s", e)
return # Connection error
finally:
if deadline is not None:
with suppress(OSError):
sock.settimeout(original_timeout)
# Read more data
try:
data = sock.recv(RECV_BUFFER)
if not data:
return # Connection closed
self._buffer += data.decode("utf-8", errors="replace")
except TimeoutError:
continue # Keep waiting
except OSError as e:
logger.warning("Socket error: %s", e)
return # Connection error
def _parse_message(self, line: str) -> IRCMessage:
"""Parse an IRC message line into components.
@@ -460,14 +427,9 @@ class IRCClient:
return False
return True
def read_messages(
self,
*,
auto_handle: bool = True,
deadline: float | None = None,
) -> Iterator[IRCMessage]:
def read_messages(self, *, auto_handle: bool = True) -> Iterator[IRCMessage]:
"""Read and yield IRC messages, optionally auto-handling PING/VERSION."""
for line in self._recv_lines(deadline):
for line in self._recv_lines():
msg = self._parse_message(line)
# Auto-handle certain events
@@ -491,9 +453,13 @@ class IRCClient:
) -> DCCOffer | None:
"""Wait for a DCC SEND offer. Returns None on timeout or no results."""
target_event = IRCEvent.SEARCH_RESULT if result_type else IRCEvent.BOOK_RESULT
deadline = time.time() + timeout
start = time.time()
for msg in self.read_messages():
if time.time() - start > timeout:
logger.warning("Timeout waiting for DCC offer")
return None
for msg in self.read_messages(deadline=deadline):
if msg.event == target_event:
if not self._is_allowed_dcc_sender(msg, expected_senders):
continue
@@ -525,8 +491,6 @@ class IRCClient:
count = match.group(1)
logger.info("Found %s matches", count)
if time.time() >= deadline:
logger.warning("Timeout waiting for DCC offer")
return None
@property
+2 -41
View File
@@ -72,10 +72,7 @@ def irc_settings() -> list[SettingsField]:
key="IRC_CHANNEL",
label="Channel",
placeholder="e.g. ebooks",
description=(
"Channel name without the # prefix. Used for all searches unless a "
"separate audiobook channel is configured below."
),
description="Channel name without the # prefix",
required=True,
env_supported=True,
),
@@ -91,43 +88,7 @@ def irc_settings() -> list[SettingsField]:
key="IRC_SEARCH_BOT",
label="Search bot",
placeholder="e.g. search",
description=(
"The search bot to address queries to (required). Searches are sent as "
'"@<bot> <query>".'
),
required=True,
env_supported=True,
),
HeadingField(
key="audiobook_heading",
title="Audiobooks",
description=(
"Some networks index audiobooks in a separate channel from ebooks "
"(for example #ebooks for ebooks and #bookz for audiobooks). "
"Configure that channel here to search it for audiobook requests. "
"Leave these blank to search the main channel above for both."
),
),
TextField(
key="IRC_AUDIOBOOK_CHANNEL",
label="Audiobook channel",
placeholder="e.g. bookz",
description=(
"Optional. Channel name (without the # prefix) to use for audiobook "
"searches. Leave blank to use the main channel above for audiobooks too."
),
required=False,
env_supported=True,
),
TextField(
key="IRC_AUDIOBOOK_SEARCH_BOT",
label="Audiobook search bot",
placeholder="e.g. search",
description=(
"Optional. Search bot for the audiobook channel. Leave blank to reuse "
"the main search bot above. Only used when an audiobook channel is set."
),
required=False,
description="The search bot to query for results",
env_supported=True,
),
HeadingField(
+41 -147
View File
@@ -15,7 +15,6 @@ if TYPE_CHECKING:
from shelfmark.api.websocket import ws_manager
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import is_audiobook
from shelfmark.release_sources import (
ColumnColorHint,
ColumnRenderType,
@@ -89,17 +88,6 @@ def _emit_status(message: str, phase: str = "searching") -> None:
MIN_SEARCH_INTERVAL = 15.0
_last_search_time: float = 0
# Anti-spam budget: the exact same message may only be posted to the channel a limited
# number of times within a rolling window. This stops a retry/refresh loop from flooding
# the channel with the same line over and over, while still allowing a few genuine retries
# (a search that came back empty can be tried again, and Refresh works until the budget runs
# out). Normal use never hits this: successful searches are served from the result cache
# without re-posting at all.
MAX_IDENTICAL_SENDS = 3
IDENTICAL_SEND_WINDOW_SECONDS = 24 * 60 * 60 # 24 hours
# message-send-key -> timestamps of recent posts of that exact message
_recent_message_sends: dict[str, list[float]] = {}
def _enforce_rate_limit() -> None:
"""Ensure minimum time between searches."""
@@ -114,36 +102,6 @@ def _enforce_rate_limit() -> None:
_last_search_time = time.time()
def _query_identity(server: str, channel: str, query: str) -> str:
"""Stable identity for a query on a given IRC server-channel.
Used as BOTH the result-cache key and the per-query send-counter key, so the same
query shares one cached answer and one send budget regardless of which book or
content type triggered it.
"""
return f"{server.casefold()}:{channel.casefold()}:{query.strip().casefold()}"
def _recent_send_count(key: str) -> int:
"""Number of times this exact message was posted within the rolling window."""
cutoff = time.time() - IDENTICAL_SEND_WINDOW_SECONDS
timestamps = [ts for ts in _recent_message_sends.get(key, []) if ts > cutoff]
if timestamps:
_recent_message_sends[key] = timestamps
else:
_recent_message_sends.pop(key, None)
return len(timestamps)
def _record_message_sent(key: str) -> None:
"""Record that an exact message was just posted to the channel."""
now = time.time()
cutoff = now - IDENTICAL_SEND_WINDOW_SECONDS
timestamps = [ts for ts in _recent_message_sends.get(key, []) if ts > cutoff]
timestamps.append(now)
_recent_message_sends[key] = timestamps
@register_source("irc")
class IRCReleaseSource(ReleaseSource):
"""Search IRC channels for ebook and audiobook releases."""
@@ -159,16 +117,11 @@ class IRCReleaseSource(ReleaseSource):
self._online_servers: set[str] | None = None
def is_available(self) -> bool:
"""Check if IRC is configured (server, channel, nick, and search bot are set).
The search bot is required: without it we would post bare queries straight
to the channel, which reads as spam and gets the nick banned.
"""
"""Check if IRC is configured (server, channel, and nick are set)."""
server = _config_text("IRC_SERVER")
channel = _config_text("IRC_CHANNEL")
nick = _config_text("IRC_NICK")
search_bot = _config_text("IRC_SEARCH_BOT")
return bool(server and channel and nick and search_bot)
return bool(server and channel and nick)
def get_column_config(self) -> ReleaseColumnConfig:
"""Configure UI columns for IRC results."""
@@ -226,12 +179,25 @@ class IRCReleaseSource(ReleaseSource):
logger.debug("IRC source is disabled, skipping search")
return []
# Check cache first (unless expand_search/refresh is requested)
if not expand_search:
cached = get_cached_results(book.provider, book.provider_id, content_type=content_type)
if cached:
_emit_status("Using cached results", phase="complete")
self._online_servers = set(cached.get("online_servers", []))
return cached["releases"]
# Build search query
query = plan.primary_query or self._build_query(book)
if not query:
logger.warning("No search query could be built")
return []
logger.info("IRC search: %s", query)
# Enforce rate limit
_enforce_rate_limit()
# Get IRC settings
server = _config_text("IRC_SERVER")
port = _config_port("IRC_PORT", 6697)
@@ -240,67 +206,6 @@ class IRCReleaseSource(ReleaseSource):
nick = _config_text("IRC_NICK")
search_bot = _config_text("IRC_SEARCH_BOT")
# Audiobooks may be indexed in a separate channel from ebooks on some networks
# (e.g. #ebooks for ebooks, #bookz for audiobooks). When an audiobook channel is
# configured and an audiobook was requested, route the search there (with its own
# search bot if set). Otherwise fall back to the main channel/bot, which keeps the
# single-channel networks that index both formats working unchanged.
if is_audiobook(content_type):
audiobook_channel = _config_text("IRC_AUDIOBOOK_CHANNEL")
if audiobook_channel:
channel = audiobook_channel
audiobook_search_bot = _config_text("IRC_AUDIOBOOK_SEARCH_BOT")
if audiobook_search_bot:
search_bot = audiobook_search_bot
# Never post an unaddressed query to the channel. A bare book title looks like
# spam to everyone else in the channel and gets the nick banned. Searches must
# be addressed to a search bot ("@<bot> <query>").
if not search_bot:
logger.warning(
"IRC search bot not configured; refusing to post unaddressed query to channel"
)
_emit_status("IRC search bot not configured", phase="error")
return []
# One identity per query on this server-channel. The result cache and the send
# counter are both keyed on it: the SAME query shares one cached answer and one
# send budget regardless of which book/content type triggered it, while different
# queries are independent (searching 100 different books posts 100 messages).
requested = "audiobook" if is_audiobook(content_type) else "ebook"
query_key = _query_identity(server, channel, query)
# Serve the cached whole answer for an identical query (unless this is a refresh).
if not expand_search:
cached = get_cached_results(query_key)
if cached:
_emit_status("Using cached results", phase="complete")
self._online_servers = set(cached.get("online_servers", []))
return self._filter_by_content_type(cached["releases"], requested)
# Anti-spam cap: the exact same query may only be POSTED a limited number of times
# per window, even via refresh. Beyond that, serve whatever is cached rather than
# re-posting the identical message to the channel.
if _recent_send_count(query_key) >= MAX_IDENTICAL_SENDS:
logger.info(
"IRC query hit %s-send limit in window, not re-posting: %s",
MAX_IDENTICAL_SENDS,
query,
)
_emit_status(
"Search limit reached for this query — showing latest results", phase="complete"
)
cached = get_cached_results(query_key)
if cached:
self._online_servers = set(cached.get("online_servers", []))
return self._filter_by_content_type(cached["releases"], requested)
return []
logger.info("IRC search: %s", query)
# Enforce rate limit
_enforce_rate_limit()
client = None
try:
# Get or reuse IRC connection
@@ -316,27 +221,28 @@ class IRCReleaseSource(ReleaseSource):
# Capture online servers (elevated users in channel)
self._online_servers = client.online_servers
# Send search request (always addressed to the search bot, never bare)
client.send_message(f"#{channel}", f"@{search_bot} {query}")
_record_message_sent(query_key)
# Send search request
search_msg = f"@{search_bot} {query}" if search_bot else query
client.send_message(f"#{channel}", search_msg)
# Wait for results DCC - this is the long wait.
# Don't restrict the sender to the trigger bot's nick: many channels answer an
# "@search" from a differently-named results bot. The DCC endpoint/filename are
# still validated, and wait_for_dcc falls back to the channel's server list.
# Wait for results DCC - this is the long wait
_emit_status(f"Connected to #{channel} - Waiting for results...", phase="searching")
offer = client.wait_for_dcc(timeout=60.0, result_type=True)
online_servers = list(self._online_servers) if self._online_servers else None
wait_kwargs = {"expected_senders": {search_bot}} if search_bot else {}
offer = client.wait_for_dcc(timeout=60.0, result_type=True, **wait_kwargs)
if not offer:
logger.info("No search results received")
_emit_status("No results found", phase="complete")
# Release connection for reuse (don't close it)
connection_manager.release_connection(client)
# Cache the (empty) answer under the query identity so an identical query
# is served from cache instead of re-posting.
cache_results(query_key, query, [], online_servers=online_servers)
# Cache empty result to avoid repeated failed searches
cache_results(
book.provider,
book.provider_id,
book.title,
[],
content_type=content_type,
online_servers=list(self._online_servers) if self._online_servers else None,
)
return []
# Download results file
@@ -354,22 +260,19 @@ class IRCReleaseSource(ReleaseSource):
# Release connection for reuse (don't close it)
connection_manager.release_connection(client)
# A single "@search" returns one file containing every format. Parse the whole
# answer (both ebooks and audiobooks) and cache it under the query identity, so
# requesting the other content type is served from cache without re-posting.
ebook_releases = self._convert_to_releases(
parse_results_file(content, content_type="ebook"), content_type="ebook"
)
audiobook_releases = self._convert_to_releases(
parse_results_file(content, content_type="audiobook"), content_type="audiobook"
)
# Convert to Release objects
results = parse_results_file(content, content_type=content_type)
releases = self._convert_to_releases(results, content_type=content_type)
# Cache results
cache_results(
query_key,
query,
ebook_releases + audiobook_releases,
online_servers=online_servers,
book.provider,
book.provider_id,
book.title,
releases,
content_type=content_type,
online_servers=list(self._online_servers) if self._online_servers else None,
)
releases = audiobook_releases if requested == "audiobook" else ebook_releases
except DCCError as e:
logger.exception("DCC error during search")
@@ -487,15 +390,6 @@ class IRCReleaseSource(ReleaseSource):
return releases
@staticmethod
def _filter_by_content_type(releases: list[Release], requested: str) -> list[Release]:
"""Pick the requested content type out of a cached whole answer.
The cache stores releases for every content type under one query identity; each
release is tagged with its content type (defaulting to ebook when missing).
"""
return [release for release in releases if (release.content_type or "ebook") == requested]
@staticmethod
def _parse_size(size_str: str) -> int | None:
"""Parse human-readable size (e.g., '1.2MB', '500K') to bytes."""
+9 -37
View File
@@ -124,12 +124,10 @@ class ProwlarrClient:
msg = f"Invalid JSON response: {e}"
raise ValueError(msg) from e
except requests.exceptions.HTTPError as e:
status_code = e.response.status_code if e.response is not None else "unknown"
reason = e.response.reason if e.response is not None else "unknown"
logger.exception(
"Prowlarr API HTTP error: %s %s",
status_code,
reason,
e.response.status_code,
e.response.reason,
)
raise
except requests.exceptions.RequestException:
@@ -158,55 +156,36 @@ class ProwlarrClient:
logger.info("Prowlarr connection successful: version %s", version)
return True, f"Connected to Prowlarr {version}"
def get_indexers(self, *, raise_on_error: bool = False) -> list[dict[str, Any]]:
"""Get all configured indexers.
Args:
raise_on_error: When True, propagate API failures instead of
returning an empty list. Callers that must distinguish
"no indexers" from "the request failed" should set this.
"""
def get_indexers(self) -> list[dict[str, Any]]:
"""Get all configured indexers."""
try:
return _normalize_json_object_list(
self._request("GET", "/api/v1/indexer"),
context="Prowlarr indexer list",
)
except _PROWLARR_CLIENT_ERRORS:
if raise_on_error:
raise
logger.exception("Failed to get indexers")
return []
def get_enabled_indexers_detailed(
self, *, raise_on_error: bool = False
) -> list[dict[str, Any]]:
def get_enabled_indexers_detailed(self) -> list[dict[str, Any]]:
"""Get enabled indexers, including implementation metadata.
Note: Prowlarr indexer "name" is user-configurable; prefer
"implementation"/"implementationName" for stable identification.
"""
indexers = self.get_indexers(raise_on_error=raise_on_error)
indexers = self.get_indexers()
return [idx for idx in indexers if idx.get("enable", False)]
def get_enriched_indexer_ids(
self,
*,
restrict_to: list[int] | None = None,
indexers: list[dict[str, Any]] | None = None,
) -> list[int]:
def get_enriched_indexer_ids(self, *, restrict_to: list[int] | None = None) -> list[int]:
"""Return enabled indexer IDs that benefit from extra Torznab handling.
Args:
restrict_to: Optional list of candidate indexer IDs to consider.
indexers: Optional already-fetched enabled indexer list, so callers
that need the full records for other reasons can avoid a second
round trip.
"""
enriched_ids: list[int] = []
for idx in indexers if indexers is not None else self.get_enabled_indexers_detailed():
for idx in self.get_enabled_indexers_detailed():
idx_id_int = coerce_int_like(idx.get("id"))
if idx_id_int is None:
continue
@@ -233,17 +212,10 @@ class ProwlarrClient:
Prowlarr exposes seedTime in minutes, which is also the unit expected by
torrent clients.
Raises:
requests.exceptions.RequestException (and other client errors) when
the indexer list cannot be fetched. An empty dict strictly means
"no share limits are configured", never "the request failed" -
callers rely on this to avoid silently dropping seed limits.
"""
settings_by_indexer: dict[int, IndexerSeedSettings] = {}
for idx in self.get_enabled_indexers_detailed(raise_on_error=True):
for idx in self.get_enabled_indexers_detailed():
idx_id_int = coerce_int_like(idx.get("id"))
if idx_id_int is None:
continue
+16 -200
View File
@@ -1,15 +1,10 @@
"""Prowlarr download handler - resolves releases and delegates lifecycle to shared clients."""
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
import requests
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import normalize_optional_text
from shelfmark.core.search_plan import build_release_search_plan
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.clients import (
DownloadClient,
get_client,
@@ -28,13 +23,9 @@ from shelfmark.download.clients.base_handler import (
DownloadRequest,
ExternalClientHandler,
)
from shelfmark.metadata_providers import BookMetadata
from shelfmark.release_sources import register_handler
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
from shelfmark.release_sources.prowlarr.cache import cache_release, get_release, remove_release
from shelfmark.release_sources.prowlarr.source import ProwlarrSource
from shelfmark.release_sources.prowlarr.cache import get_release, remove_release
from shelfmark.release_sources.prowlarr.utils import (
build_source_id,
coerce_int_like,
get_preferred_download_url,
get_protocol,
@@ -46,16 +37,6 @@ if TYPE_CHECKING:
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
# Errors that ProwlarrClient can raise when fetching indexer settings.
_SEED_SETTINGS_FALLBACK_ERRORS = (
requests.exceptions.RequestException,
OSError,
RuntimeError,
TypeError,
ValueError,
)
__all__ = [
"ProwlarrHandler",
"POLL_INTERVAL",
@@ -68,11 +49,6 @@ __all__ = [
POLL_INTERVAL = _DEFAULT_POLL_INTERVAL
COMPLETED_PATH_RETRY_INTERVAL = _DEFAULT_COMPLETED_PATH_RETRY_INTERVAL
COMPLETED_PATH_MAX_ATTEMPTS = _DEFAULT_COMPLETED_PATH_MAX_ATTEMPTS
EXPIRED_LINK_REFRESH_ERROR = (
"The indexer download link expired and the release could not be refreshed. "
"Search again for a fresh result."
)
HASH_DETECTION_ERROR = "Could not determine torrent hash from URL"
def _coerce_positive_minutes(raw_minutes: object) -> int | None:
@@ -86,47 +62,6 @@ def _coerce_positive_minutes(raw_minutes: object) -> int | None:
class ProwlarrHandler(ExternalClientHandler):
"""Handler for Prowlarr downloads via configured torrent or usenet client."""
@staticmethod
def _build_prowlarr_client() -> ProwlarrClient | None:
"""Build a ProwlarrClient from config, or None if not configured."""
raw_url = config.get("PROWLARR_URL", "")
raw_api_key = config.get("PROWLARR_API_KEY", "")
url = normalize_optional_text(raw_url) if isinstance(raw_url, str) else None
api_key = normalize_optional_text(raw_api_key) if isinstance(raw_api_key, str) else None
if not url or not api_key:
return None
normalized_url = normalize_http_url(url)
if not normalized_url:
return None
return ProwlarrClient(normalized_url, api_key)
def _fetch_seed_settings_fallback(self, raw_indexer_id: object) -> IndexerSeedSettings | None:
"""Fetch share limits for one indexer directly from Prowlarr.
Used when the cached release is missing its search-time seed-limit
enrichment so that transient failures during search cannot cause a
torrent to be added without its configured share limits.
"""
indexer_id = coerce_int_like(raw_indexer_id)
if indexer_id is None:
return None
client = self._build_prowlarr_client()
if client is None:
return None
try:
settings = client.get_indexer_seed_settings(restrict_to=[indexer_id])
except _SEED_SETTINGS_FALLBACK_ERRORS:
logger.warning(
"Grab-time seed settings fallback failed for indexerId=%s",
indexer_id,
exc_info=True,
)
return None
return settings.get(indexer_id)
def _get_client(self, protocol: str) -> DownloadClient | None:
"""Compatibility shim so module-level patching still works in tests."""
return get_client(protocol)
@@ -146,30 +81,18 @@ class ProwlarrHandler(ExternalClientHandler):
def build_retry_resolution_fields(self, release_data: dict[str, Any]) -> dict[str, Any]:
source_id = normalize_optional_text(release_data.get("source_id"))
extra = release_data.get("extra")
if not isinstance(extra, dict):
extra = {}
if source_id is None:
return {}
retry_source_context: dict[str, Any] = {}
indexer_id = release_data.get("indexer_id") or extra.get("indexer_id")
if indexer_id is not None:
retry_source_context["indexer_id"] = indexer_id
indexer = normalize_optional_text(release_data.get("indexer") or extra.get("indexer"))
if indexer is not None and indexer.lower() != "unknown":
retry_source_context["indexer"] = indexer
info_url = normalize_optional_text(release_data.get("info_url") or extra.get("info_url"))
if info_url is not None:
retry_source_context["info_url"] = info_url
if source_id is not None:
retry_source_context["source_id"] = source_id
prowlarr_result = get_release(source_id)
if prowlarr_result is None:
return {}
return {
"retry_download_url": None,
"retry_download_protocol": None,
"retry_source_context": retry_source_context,
"retry_download_url": normalize_optional_text(
get_preferred_download_url(prowlarr_result)
),
"retry_download_protocol": normalize_optional_text(get_protocol(prowlarr_result)),
}
@classmethod
@@ -216,12 +139,13 @@ class ProwlarrHandler(ExternalClientHandler):
# Look up the cached release
prowlarr_result = get_release(task.task_id)
if not prowlarr_result:
logger.info("Prowlarr release cache miss, refreshing: %s", task.task_id)
prowlarr_result = self._refresh_release(task)
if prowlarr_result is None:
logger.warning("Prowlarr release refresh failed: %s", task.task_id)
status_callback("error", EXPIRED_LINK_REFRESH_ERROR)
restored_request = self._restore_download_request_from_task(task)
if restored_request is None:
logger.warning("Release cache miss: %s", task.task_id)
status_callback("error", "Release not found in cache (may have expired)")
return None
logger.info("Restored Prowlarr download request for retry: %s", task.task_id)
return restored_request
# Extract download URL
download_url = get_preferred_download_url(prowlarr_result)
@@ -247,28 +171,6 @@ class ProwlarrHandler(ExternalClientHandler):
seeding_time_limit = _coerce_positive_minutes(raw_configured_seed_time)
ratio_limit = float(raw_configured_ratio) if raw_configured_ratio is not None else None
# Fallback: search-time enrichment can be missing when the indexer
# settings fetch transiently failed during the search (#795).
# Re-resolve the limits from Prowlarr at grab time so torrents are
# never sent to the client without their configured share limits.
if seeding_time_limit is None and ratio_limit is None and protocol == "torrent":
fallback = self._fetch_seed_settings_fallback(prowlarr_result.get("indexerId"))
if fallback:
seeding_time_limit = _coerce_positive_minutes(
fallback.get("seeding_time_limit_minutes")
)
raw_ratio = fallback.get("ratio_limit")
ratio_limit = float(raw_ratio) if raw_ratio is not None else None
if seeding_time_limit is None and ratio_limit is None and protocol == "torrent":
logger.warning(
"Prowlarr seed preferences are enabled but no share limits "
"could be resolved for release '%s' (indexerId=%s); the "
"torrent will use the client's global limits",
release_name,
prowlarr_result.get("indexerId"),
)
return DownloadRequest(
url=download_url,
protocol=protocol,
@@ -278,92 +180,6 @@ class ProwlarrHandler(ExternalClientHandler):
ratio_limit=ratio_limit,
)
def _refresh_release(self, task: DownloadTask) -> dict[str, Any] | None:
"""Re-query Prowlarr and cache the exact original release if it still exists."""
title = normalize_optional_text(task.title)
if title is None:
return None
context = getattr(task, "retry_source_context", None)
if not isinstance(context, dict):
context = {}
indexer = normalize_optional_text(context.get("indexer"))
book = BookMetadata(
provider="shelfmark",
provider_id=task.task_id,
title=title,
authors=[task.author] if task.author else [],
search_title=title,
search_author=task.author,
)
plan = build_release_search_plan(
book,
indexers=[indexer] if indexer is not None else None,
)
source = ProwlarrSource()
results = source.search(book, plan, content_type=task.content_type or "ebook")
for release in results:
raw_release = get_release(release.source_id)
if raw_release is None:
continue
if not self._raw_release_matches_task(raw_release, task.task_id):
continue
cache_release(task.task_id, raw_release)
logger.info("Refreshed Prowlarr release: %s", task.task_id)
return raw_release
return None
@staticmethod
def _raw_release_matches_task(raw_release: dict[str, Any], task_id: str) -> bool:
wanted = normalize_optional_text(task_id)
if wanted is None:
return False
bare = [
identity
for identity in (
normalize_optional_text(raw_release.get("guid")),
normalize_optional_text(raw_release.get("infoUrl")),
)
if identity is not None
]
identities = [*bare, build_source_id(raw_release)]
indexer_id = coerce_int_like(raw_release.get("indexerId"))
if indexer_id is not None:
identities.extend(f"{indexer_id}:{identity}" for identity in bare)
return wanted in identities
def _refresh_download_request_after_add_failure(
self,
*,
task: DownloadTask,
request: DownloadRequest,
error: Exception,
status_callback: Callable[[str, str | None], None],
) -> DownloadRequest | None:
"""Refresh once when a cached Prowlarr torrent proxy URL has expired."""
if request.protocol != "torrent":
return None
if HASH_DETECTION_ERROR not in str(error):
return None
parsed = urlparse(request.url)
if parsed.scheme.lower() not in {"http", "https"}:
return None
logger.info("Refreshing stale Prowlarr torrent URL for %s", task.task_id)
remove_release(task.task_id)
refreshed_request = self._resolve_download(task, status_callback)
if refreshed_request is None:
raise RuntimeError(EXPIRED_LINK_REFRESH_ERROR) from error
return refreshed_request
def _on_download_complete(self, task: DownloadTask) -> None:
"""Remove completed release from the Prowlarr cache."""
remove_release(task.task_id)
@@ -190,18 +190,6 @@ def prowlarr_config_settings() -> list[SettingsField]:
description="Automatically retry search without category filtering if no results are found",
show_when={"field": "PROWLARR_ENABLED", "value": True},
),
CheckboxField(
key="PROWLARR_COLLAPSE_DUPLICATES",
label="Show one row per release",
default=True,
description=(
"Collapse a release that several indexer entries returned down to a single row, "
"keeping the entry with the best Prowlarr priority. Turn this off to see every "
"entry that carried it, which is what makes results from filter-specific entries "
"(freeleech and the like) visible."
),
show_when={"field": "PROWLARR_ENABLED", "value": True},
),
CheckboxField(
key="PROWLARR_USE_SEED_PREFERENCES",
label="Use Prowlarr seed preferences",
+63 -193
View File
@@ -2,17 +2,13 @@
import re
import time
from threading import Lock
from typing import TYPE_CHECKING, ClassVar, NoReturn
import requests
if TYPE_CHECKING:
from shelfmark.core.search_plan import ReleaseSearchPlan
from shelfmark.metadata_providers import BookMetadata
from shelfmark.core.config import config
from shelfmark.core.languages import normalize_language
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import normalize_optional_text
from shelfmark.core.search_plan import ReleaseSearchVariant
@@ -34,7 +30,6 @@ from shelfmark.release_sources import (
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
from shelfmark.release_sources.prowlarr.cache import cache_release
from shelfmark.release_sources.prowlarr.utils import (
build_source_id,
coerce_float_like,
coerce_int_like,
get_protocol,
@@ -46,14 +41,6 @@ _SIZE_UNIT_BASE = 1024
_TWO_FORMATS = 2
_PROWLARR_SOURCE_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, ValueError)
# Prowlarr indexer priority is 1-50 and lower is preferred; unknown sorts last.
_UNRANKED_INDEXER_RANK = 51
# Errors that can surface from ProwlarrClient.get_indexer_seed_settings(). The
# client raises requests exceptions (subclasses of OSError via IOError lineage
# is not guaranteed), so include RequestException explicitly.
_PROWLARR_SEED_SETTINGS_ERRORS = (*_PROWLARR_SOURCE_ERRORS, requests.exceptions.RequestException)
def _raise_timeout_error(message: str) -> NoReturn:
raise TimeoutError(message)
@@ -74,118 +61,6 @@ def _coerce_indexer_id(value: object) -> int | None:
return coerce_int_like(value)
def _identity_text(value: object) -> str | None:
"""Trimmed text for an identity field, or None when there is nothing usable."""
if isinstance(value, str):
return value.strip() or None
if isinstance(value, (int, float)) and not isinstance(value, bool):
return str(value)
return None
def _release_identity(result: dict) -> str | None:
"""Identify the underlying release, independent of which indexer surfaced it.
Strong identifiers only. Title is deliberately excluded because matching on
it here would merge two genuinely different releases that happen to share a
name, and every caller of this either drops or overwrites a row on a match.
Returns None when nothing identifies the result.
"""
for field in ("guid", "downloadUrl", "magnetUrl", "infoUrl"):
identity = _identity_text(result.get(field))
if identity is not None:
return identity
return None
def _result_dedup_key(result: dict) -> tuple[int | None, str] | None:
"""Dedup key for a raw Prowlarr result, or None if it cannot be identified.
One tracker is often configured in Prowlarr as several indexer entries that
differ only by a server-side search filter, say a "freeleech only" entry
alongside an unfiltered one. Those entries return the same guid for the same
torrent, so keying on the guid alone throws away the filtered entry's copy
and with it the only signal that the release matched the filter. Including
the indexer id keeps the entries distinct.
Title is an acceptable last resort here, unlike in _release_identity, because
the indexer id is part of the key: it only ever collapses a literal repeat
from one indexer, never two rows from different entries.
"""
identity = _release_identity(result) or _identity_text(result.get("title"))
if identity is None:
return None
return (_coerce_indexer_id(result.get("indexerId")), identity)
def _build_indexer_priority(indexers: list[dict]) -> dict[int, int]:
"""Map indexer id to the priority configured in Prowlarr. Lower is preferred.
Users already rank their indexers in Prowlarr, and on trackers configured as
several entries that ranking is usually the meaningful one: a "freeleech
only" entry is typically given a better priority than the unfiltered entry
beside it. Reusing it avoids asking for the same ordering a second time.
"""
priority: dict[int, int] = {}
for indexer in indexers:
indexer_id = _coerce_indexer_id(indexer.get("id"))
if indexer_id is None:
continue
rank = coerce_int_like(indexer.get("priority"))
if rank is not None:
priority[indexer_id] = rank
return priority
def _rank_for_indexer_id(indexer_id: object, priority: dict[int, int]) -> int:
"""Preference rank for an indexer id. Lower wins, unknown ranks last."""
coerced = _coerce_indexer_id(indexer_id)
if coerced is None:
return _UNRANKED_INDEXER_RANK
return priority.get(coerced, _UNRANKED_INDEXER_RANK)
def _indexer_rank(result: dict, priority: dict[int, int]) -> int:
"""Preference rank of the indexer that surfaced a raw result."""
return _rank_for_indexer_id(result.get("indexerId"), priority)
def _release_indexer_rank(release: Release, priority: dict[int, int]) -> int:
"""Preference rank of the indexer that surfaced a converted release."""
return _rank_for_indexer_id(release.extra.get("indexer_id"), priority)
def _collapse_duplicate_indexer_results(
results: list[dict], priority: dict[int, int]
) -> list[dict]:
"""Reduce a release to a single row, keeping the preferred indexer entry.
Opt-in behaviour for users who want one row per torrent. Ties keep the
result that was queried first, and the winner holds the loser's position so
the overall result order stays stable.
"""
position_by_identity: dict[str, int] = {}
kept: list[dict] = []
for result in results:
identity = _release_identity(result)
if identity is None:
kept.append(result)
continue
existing_position = position_by_identity.get(identity)
if existing_position is None:
position_by_identity[identity] = len(kept)
kept.append(result)
continue
if _indexer_rank(result, priority) < _indexer_rank(kept[existing_position], priority):
kept[existing_position] = result
return kept
def _parse_size(size_bytes: int | None) -> str | None:
"""Convert bytes to human-readable size string."""
if size_bytes is None or size_bytes <= 0:
@@ -227,6 +102,50 @@ AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "o
# Combined list for format detection (audiobook formats first for priority)
ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
# Map 3-char MAM language codes to 2-char ISO codes used by frontend color maps
MAM_LANGUAGE_MAP = {
"eng": "en",
"ita": "it",
"spa": "es",
"fra": "fr",
"fre": "fr",
"ger": "de",
"deu": "de",
"por": "pt",
"rus": "ru",
"jpn": "ja",
"jap": "ja",
"chi": "zh",
"zho": "zh",
"dut": "nl",
"nld": "nl",
"swe": "sv",
"nor": "no",
"dan": "da",
"fin": "fi",
"pol": "pl",
"cze": "cs",
"ces": "cs",
"hun": "hu",
"kor": "ko",
"ara": "ar",
"heb": "he",
"tur": "tr",
"gre": "el",
"ell": "el",
"hin": "hi",
"tha": "th",
"vie": "vi",
"ind": "id",
"ukr": "uk",
"rom": "ro",
"ron": "ro",
"bul": "bg",
"cat": "ca",
"hrv": "hr",
"slv": "sl",
"srp": "sr",
}
# Backend safeguard: cap total Prowlarr search time per request.
PROWLARR_SEARCH_TIMEOUT_SECONDS = 120.0
@@ -274,9 +193,8 @@ def _extract_mam_language(raw_title: str) -> str | None:
for token in tokens:
lang_code = token.lower()
resolved = normalize_language(lang_code)
if resolved is not None:
return resolved
if lang_code in MAM_LANGUAGE_MAP:
return MAM_LANGUAGE_MAP[lang_code]
return None
@@ -461,7 +379,8 @@ def _prowlarr_result_to_release(
formats_display = _formats_display(formats)
language_detected = _extract_mam_language(str(raw_title or ""))
source_id = build_source_id(result)
# Build the source_id from GUID or generate from indexer + title
source_id = result.get("guid") or f"{indexer}:{hash(raw_title)}"
# Cache the raw Prowlarr result so handler can look it up by source_id
cache_release(source_id, result)
@@ -524,36 +443,6 @@ def _prowlarr_result_to_release(
)
# Last successfully fetched per-indexer share limits. Used as a fallback when
# a transient Prowlarr API failure prevents fetching fresh settings during a
# search, so results are never silently cached without seed limits (#795).
_seed_settings_lock = Lock()
_last_known_seed_settings: dict[int, IndexerSeedSettings] = {}
def _fetch_indexer_seed_settings(
client: ProwlarrClient,
indexer_ids: list[int] | None,
) -> dict[int, IndexerSeedSettings]:
"""Fetch per-indexer share limits, falling back to last-known-good on failure."""
try:
fetched = client.get_indexer_seed_settings(restrict_to=indexer_ids)
except _PROWLARR_SEED_SETTINGS_ERRORS:
with _seed_settings_lock:
fallback = dict(_last_known_seed_settings)
logger.warning(
"Failed to fetch Prowlarr indexer seed settings; "
"falling back to last known settings for %s indexer(s)",
len(fallback),
exc_info=True,
)
return fallback
with _seed_settings_lock:
_last_known_seed_settings.update(fetched)
return fetched
def _apply_indexer_seed_settings(
result: dict,
indexer_seed_settings: dict[int, IndexerSeedSettings],
@@ -685,11 +574,6 @@ class ProwlarrSource(ReleaseSource):
],
extra_sort_options=[
SortOption(label="Peers", sort_key="seeders"),
SortOption(
label="Indexer priority",
sort_key="extra.indexer_priority",
default_direction="asc",
),
],
grid_template="minmax(0,2fr) minmax(140px,1fr) 50px 50px 90px 80px",
leading_cell=LeadingCellConfig(
@@ -895,15 +779,11 @@ class ProwlarrSource(ReleaseSource):
try:
auto_expand_enabled = config.get("PROWLARR_AUTO_EXPAND", False)
deadline = time.monotonic() + PROWLARR_SEARCH_TIMEOUT_SECONDS
enabled_indexers = client.get_enabled_indexers_detailed()
indexer_priority = _build_indexer_priority(enabled_indexers)
# Some indexers benefit from title+author queries and extra format detection.
enriched_indexer_ids = client.get_enriched_indexer_ids(
restrict_to=indexer_ids, indexers=enabled_indexers
)
enriched_indexer_ids = client.get_enriched_indexer_ids(restrict_to=indexer_ids)
enriched_indexer_ids_set = set(enriched_indexer_ids)
indexer_seed_settings = (
_fetch_indexer_seed_settings(client, indexer_ids)
client.get_indexer_seed_settings(restrict_to=indexer_ids)
if config.get("PROWLARR_USE_SEED_PREFERENCES", False)
else {}
)
@@ -941,7 +821,7 @@ class ProwlarrSource(ReleaseSource):
return results
seen_keys: set[tuple[int | None, str]] = set()
seen_keys: set[str] = set()
all_results: list[dict] = []
for idx, variant in enumerate(variants, start=1):
@@ -969,21 +849,17 @@ class ProwlarrSource(ReleaseSource):
self.last_search_type = "expanded"
for r in raw_results:
key = _result_dedup_key(r)
if key is not None:
if key in seen_keys:
continue
seen_keys.add(key)
all_results.append(r)
if config.get("PROWLARR_COLLAPSE_DUPLICATES", True):
before_collapse = len(all_results)
all_results = _collapse_duplicate_indexer_results(all_results, indexer_priority)
if len(all_results) != before_collapse:
logger.debug(
"Prowlarr: collapsed %s duplicate result(s) across indexer entries",
before_collapse - len(all_results),
key = (
r.get("guid")
or r.get("downloadUrl")
or r.get("magnetUrl")
or r.get("infoUrl")
or f"{r.get('indexerId')}:{r.get('title')}"
)
if key in seen_keys:
continue
seen_keys.add(key)
all_results.append(r)
results: list[Release] = []
enriched_source_ids: set[str] = set()
@@ -1003,19 +879,13 @@ class ProwlarrSource(ReleaseSource):
content_type,
enable_format_detection=is_enriched,
)
if idx_id_int is not None and idx_id_int in indexer_priority:
release.extra["indexer_priority"] = indexer_priority[idx_id_int]
results.append(release)
if is_enriched:
enriched_source_ids.add(release.source_id)
results.sort(
key=lambda r: (
_release_indexer_rank(r, indexer_priority),
0 if r.source_id in enriched_source_ids else 1,
)
)
# Sort results: enriched indexers first, then others
results.sort(key=lambda r: 0 if r.source_id in enriched_source_ids else 1)
if results:
torrent_count = sum(1 for r in results if r.protocol == ReleaseProtocol.TORRENT)
@@ -32,28 +32,6 @@ def coerce_int_like(value: object) -> int | None:
return int(normalized)
def build_source_id(result: dict) -> str:
"""Build the Release.source_id for a raw Prowlarr result.
Qualified by the indexer id because one tracker is often configured in
Prowlarr as several indexer entries that differ only by a server-side search
filter, and those entries return the same guid for the same torrent. Without
the qualifier the entries collide in the release cache and a grab routes
through whichever entry happened to cache last.
"""
guid = result.get("guid")
if guid:
base = str(guid)
else:
indexer = result.get("indexer", "Unknown")
base = f"{indexer}:{hash(result.get('title', 'Unknown'))}"
indexer_id = coerce_int_like(result.get("indexerId"))
if indexer_id is None:
return base
return f"{indexer_id}:{base}"
def coerce_float_like(value: object) -> float | None:
"""Return a float for float-like config/API values, else None."""
if isinstance(value, bool):
+1 -1
View File
@@ -7,5 +7,5 @@
// This script is intentionally loaded from index.html as a classic script,
// so we need to declare it as an entry point manually.
"entry": ["public/theme-init.js"],
"entry": ["public/theme-init.js"]
}
+719 -1499
View File
File diff suppressed because it is too large Load Diff
+15 -15
View File
@@ -17,24 +17,24 @@
"test:unit": "vitest run"
},
"dependencies": {
"@tailwindcss/vite": "^4.3.3",
"react": "^19.2.8",
"react-dom": "^19.2.8",
"react-router-dom": "^7.18.2",
"@tailwindcss/vite": "^4.2.4",
"react": "^19.2.6",
"react-dom": "^19.2.6",
"react-router-dom": "^7.15.0",
"socket.io-client": "^4.7.5"
},
"devDependencies": {
"@types/node": "^26.2.0",
"@types/react": "^19.2.18",
"@types/react-dom": "^19.2.4",
"@vitejs/plugin-react": "^6.0.5",
"knip": "^6.32.0",
"oxfmt": "^0.62.0",
"oxlint": "^1.77.0",
"oxlint-tsgolint": "^7.0.2001",
"@types/node": "^25.6.2",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.1",
"knip": "^6.12.1",
"oxfmt": "^0.48.0",
"oxlint": "^1.63.0",
"oxlint-tsgolint": "^0.22.1",
"tailwindcss": "^4.2.2",
"typescript": "^7.0.2",
"vite": "^8.2.1",
"vitest": "^4.1.10"
"typescript": "^6.0.3",
"vite": "^8.0.11",
"vitest": "^4.1.5"
}
}
+3 -13
View File
@@ -766,8 +766,7 @@ function App() {
(cfg.show_combined_selector ?? true) &&
getDefaultMode('ebook') !== 'blocked' &&
getDefaultMode('audiobook') !== 'blocked';
const nextEffectiveCombinedMode =
nextCombinedModeAllowed && (combinedMode || cfg.force_combined_search);
const nextEffectiveCombinedMode = combinedMode && nextCombinedModeAllowed;
const activeConfiguredProvider =
nextEffectiveCombinedMode && metadataProviderState.configured_provider_combined
? metadataProviderState.configured_provider_combined
@@ -865,8 +864,7 @@ function App() {
const audiobookMode = getDefaultMode('audiobook');
return ebookMode !== 'blocked' && audiobookMode !== 'blocked';
}, [effectiveSearchMode, config?.show_combined_selector, getDefaultMode]);
const combinedModeLocked = combinedModeAllowed && config?.force_combined_search === true;
const effectiveCombinedMode = combinedModeAllowed && (combinedMode || combinedModeLocked);
const effectiveCombinedMode = combinedMode && combinedModeAllowed;
const effectiveCombinedState = effectiveCombinedMode ? combinedState : null;
const defaultMetadataProviderForContentType =
@@ -1098,9 +1096,6 @@ function App() {
series_name: book.series_name,
series_position: book.series_position,
subtitle: book.subtitle,
// From the release, never the book: book.language is the provider's
// canonical edition, which would mislabel a translated release.
language: release.language ?? undefined,
};
},
[],
@@ -1494,7 +1489,7 @@ function App() {
const handleCancel = async (id: string) => {
try {
await cancelDownload(id);
await Promise.all([fetchStatus(), refreshActivitySnapshot()]);
await fetchStatus();
} catch (error) {
console.error('Cancel failed:', error);
showToast('Failed to cancel/clear download', 'error');
@@ -2423,7 +2418,6 @@ function App() {
onContentTypeChange={setContentType}
allowedContentTypes={allowedContentTypes}
combinedMode={effectiveCombinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={combinedModeAllowed ? setCombinedMode : undefined}
queryTargets={queryTargets}
activeQueryTarget={effectiveActiveQueryTarget}
@@ -2505,7 +2499,6 @@ function App() {
onContentTypeChange={setContentType}
allowedContentTypes={allowedContentTypes}
combinedMode={effectiveCombinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={combinedModeAllowed ? setCombinedMode : undefined}
activeQueryField={activeQueryField}
searchMode={effectiveSearchMode}
@@ -2781,13 +2774,10 @@ function App() {
parsedParams={parsedParams}
config={config}
contentType={contentType}
combinedMode={combinedMode}
combinedModeAllowed={combinedModeAllowed}
advancedFilters={advancedFilters}
resolvedMetadataDefaultSort={resolvedMetadataDefaultSort}
resolvedMetadataSortOptions={resolvedMetadataSortOptions}
setContentType={setContentType}
setCombinedMode={setCombinedMode}
setSearchInput={setSearchInput}
setAdvancedFilters={setAdvancedFilters}
setShowAdvanced={setShowAdvanced}
@@ -58,8 +58,6 @@ const SEARCH_MODE_OPTIONS = [
},
];
const EMPTY_PROVIDERS: MetadataProviderSummary[] = [];
export const AdvancedFilters = ({
visible,
bookLanguages,
@@ -70,7 +68,7 @@ export const AdvancedFilters = ({
renderWrapper,
searchMode,
onSearchModeChange,
metadataProviders = EMPTY_PROVIDERS,
metadataProviders = [],
activeMetadataProvider,
onMetadataProviderChange,
contentType = 'ebook',
+3 -5
View File
@@ -1,5 +1,5 @@
import type { ReactNode } from 'react';
import { useCallback, useId, useLayoutEffect, useRef, useState } from 'react';
import { useCallback, useLayoutEffect, useRef, useState } from 'react';
import { createPortal } from 'react-dom';
import { useDismiss } from '../hooks/useDismiss';
@@ -55,7 +55,6 @@ export const Dropdown = ({
onOpenChange,
}: DropdownProps) => {
const [isOpen, setIsOpen] = useState(false);
const dropdownId = useId();
const containerRef = useRef<HTMLDivElement>(null);
const triggerRef = useRef<HTMLDivElement>(null);
const panelRef = useRef<HTMLDivElement>(null);
@@ -241,8 +240,8 @@ export const Dropdown = ({
<div className={widthClassName} ref={containerRef}>
{label && (
<label
htmlFor={dropdownId}
className="mb-1.5 block cursor-pointer text-xs font-medium text-gray-500 dark:text-gray-400"
className="mb-1.5 block text-xs font-medium text-gray-500 dark:text-gray-400"
onClick={toggleOpen}
>
{label}
</label>
@@ -252,7 +251,6 @@ export const Dropdown = ({
renderTrigger({ isOpen, toggle: toggleOpen })
) : (
<button
id={dropdownId}
type="button"
onClick={toggleOpen}
disabled={disabled}
@@ -147,7 +147,6 @@ export const DropdownList = ({
type="checkbox"
checked={selectedValues.includes(option.value)}
readOnly
aria-label={option.label}
className="pointer-events-none h-4 w-4 rounded-sm border-gray-300 text-sky-600 focus:ring-sky-500"
/>
)}
+9 -20
View File
@@ -62,7 +62,6 @@ interface HeaderProps {
onContentTypeChange?: (type: ContentType) => void;
allowedContentTypes?: ContentType[];
combinedMode?: boolean;
combinedModeLocked?: boolean;
onCombinedModeChange?: (enabled: boolean) => void;
queryTargets?: QueryTargetOption[];
activeQueryTarget?: string;
@@ -80,15 +79,6 @@ const applyTheme = (preference: string): void => {
document.documentElement.style.colorScheme = effective;
};
const DEFAULT_STATUS_COUNTS: ActivityStatusCounts = {
ongoing: 0,
completed: 0,
errored: 0,
pendingRequests: 0,
};
const EMPTY_ADMIN_USERS: ActingAsUserSelection[] = [];
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
export const Header = forwardRef<HeaderHandle, HeaderProps>(
(
{
@@ -108,7 +98,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onSettingsClick,
isAdmin = false,
canAccessSettings,
statusCounts = DEFAULT_STATUS_COUNTS,
statusCounts = { ongoing: 0, completed: 0, errored: 0, pendingRequests: 0 },
onLogoClick,
authRequired = false,
isAuthenticated = false,
@@ -116,7 +106,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
displayName,
actingAsUser = null,
onActingAsUserChange,
adminUsers = EMPTY_ADMIN_USERS,
adminUsers = [],
isAdminUsersLoading = false,
adminUsersError = null,
hasLoadedAdminUsers = false,
@@ -128,9 +118,8 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onContentTypeChange,
allowedContentTypes,
combinedMode,
combinedModeLocked,
onCombinedModeChange,
queryTargets = EMPTY_QUERY_TARGETS,
queryTargets = [],
activeQueryTarget = 'general',
onQueryTargetChange,
activeQueryField = null,
@@ -316,8 +305,8 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
// Determine if we should show icons only (both URLs configured)
const showIconsOnly = Boolean(calibreWebUrl && audiobookLibraryUrl);
// Icon buttons - reused for both states
const iconButtonsNode = (
// Icon buttons component - reused for both states
const IconButtons = () => (
<div className="flex items-center gap-2">
{/* Book Library Button */}
{calibreWebUrl && (
@@ -590,7 +579,6 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onClick={handleLogout}
className="hover-action shrink-0 rounded-full p-2 text-red-600 transition-colors dark:text-red-400"
title="Sign Out"
aria-label="Sign Out"
>
<svg
className="h-5 w-5"
@@ -686,7 +674,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
/>
))}
{iconButtonsNode}
<IconButtons />
</div>
{/* Search bar - appear second on mobile (below logo+icons), first on desktop (left side) */}
@@ -723,7 +711,6 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
onContentTypeChange={onContentTypeChange}
allowedContentTypes={allowedContentTypes}
combinedMode={combinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={onCombinedModeChange}
queryTargets={queryTargets}
activeQueryTarget={activeQueryTarget}
@@ -736,7 +723,9 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
{/* When search is NOT active: show icon buttons only on the right */}
{!showSearch && (
<div className="flex min-h-[48px] items-center justify-end">{iconButtonsNode}</div>
<div className="flex min-h-[48px] items-center justify-end">
<IconButtons />
</div>
)}
</div>
</header>
@@ -101,7 +101,6 @@ const PasswordLoginForm = ({
type="text"
id="username"
name="username"
aria-label="Username"
autoComplete="username"
autoCapitalize="none"
autoCorrect="off"
@@ -132,7 +131,6 @@ const PasswordLoginForm = ({
type={showPassword ? 'text' : 'password'}
id="password"
name="password"
aria-label="Password"
autoComplete="current-password"
autoCapitalize="none"
autoCorrect="off"
@@ -170,7 +168,6 @@ const PasswordLoginForm = ({
checked={rememberMe}
onChange={(event) => setRememberMe(event.target.checked)}
disabled={isLoading}
aria-label="Remember me for 7 days"
className="h-4 w-4 rounded-sm accent-sky-900 focus:ring-2 focus:ring-sky-500 disabled:cursor-not-allowed disabled:opacity-50"
style={{ borderColor: 'var(--border-color)' }}
/>
+1 -5
View File
@@ -382,11 +382,7 @@ export const ReleaseCell = ({
className={`flex items-center ${alignClass} gap-1.5 truncate text-xs text-gray-600 dark:text-gray-300`}
>
<span className={`h-2 w-2 shrink-0 rounded-full ${dotColor}`} title={protocolLabel} />
{/* Titled because one tracker can appear as several indexer entries whose
names share a prefix, and truncation would make the rows look identical */}
<span className="truncate" title={displayValue}>
{displayValue}
</span>
<span className="truncate">{displayValue}</span>
{peers && <span className="shrink-0 text-gray-400 dark:text-gray-500">{peers}</span>}
</div>
);
+2 -5
View File
@@ -729,8 +729,6 @@ function ErrorState({ message }: { message: string }) {
);
}
const EMPTY_SUPPORTED_AUDIOBOOK_FORMATS: string[] = [];
const ReleaseModalSession = ({
book,
onClose,
@@ -739,7 +737,7 @@ const ReleaseModalSession = ({
onRequestBook,
getPolicyModeForSource,
supportedFormats,
supportedAudiobookFormats = EMPTY_SUPPORTED_AUDIOBOOK_FORMATS,
supportedAudiobookFormats = [],
contentType,
defaultLanguages,
bookLanguages,
@@ -956,7 +954,7 @@ const ReleaseModalSession = ({
const fromExtra = (columnConfig.extra_sort_options || []).map((opt) => ({
label: opt.label,
sortKey: opt.sort_key,
defaultDirection: opt.default_direction ?? ('desc' as const),
defaultDirection: 'desc' as const, // Extra sort options are typically numeric (e.g., peers)
}));
return [...fromColumns, ...fromExtra];
}, [sortableColumns, columnConfig.extra_sort_options]);
@@ -2096,7 +2094,6 @@ const ReleaseModalSession = ({
value={manualQuery}
onChange={(e) => setManualQuery(e.target.value)}
placeholder="Type a custom search query (overrides all sources)"
aria-label="Custom search query"
className="w-full rounded-lg border border-(--border-muted) bg-(--bg) px-3 py-2 text-sm text-(--text)"
/>
<button
@@ -64,11 +64,9 @@ const getRequestConfirmationSessionKey = (payload: CreateRequestPayload): string
].join('|');
};
const EMPTY_PAYLOADS: CreateRequestPayload[] = [];
export function RequestConfirmationModal({
payload,
extraPayloads = EMPTY_PAYLOADS,
extraPayloads = [],
allowNotes,
onConfirm,
onClose,
@@ -91,7 +89,7 @@ export function RequestConfirmationModal({
function RequestConfirmationModalSession({
payload,
extraPayloads = EMPTY_PAYLOADS,
extraPayloads = [],
allowNotes,
onConfirm,
onClose,
@@ -307,7 +305,6 @@ function RequestConfirmationModalSession({
</label>
<textarea
id="request-note"
aria-label="Note (optional)"
value={note}
onChange={(event) => setNote(truncateRequestNote(event.target.value))}
maxLength={MAX_REQUEST_NOTE_LENGTH}
+6 -18
View File
@@ -34,7 +34,6 @@ interface SearchBarProps {
onContentTypeChange?: (type: ContentType) => void;
allowedContentTypes?: ContentType[];
combinedMode?: boolean;
combinedModeLocked?: boolean;
onCombinedModeChange?: (enabled: boolean) => void;
queryTargets?: QueryTargetOption[];
activeQueryTarget?: string;
@@ -49,7 +48,6 @@ export interface SearchBarHandle {
const EMPTY_SORT_OPTIONS: SortOption[] = [];
const EMPTY_AUTOCOMPLETE_OPTIONS: DynamicFieldOption[] = [];
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
const BookIcon = () => (
<svg
@@ -182,9 +180,8 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
onContentTypeChange,
allowedContentTypes,
combinedMode = false,
combinedModeLocked = false,
onCombinedModeChange,
queryTargets = EMPTY_QUERY_TARGETS,
queryTargets = [],
activeQueryTarget = 'general',
onQueryTargetChange,
activeQueryField,
@@ -521,7 +518,6 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
type="checkbox"
checked={Boolean(value)}
onChange={(e) => onChange(e.target.checked)}
aria-label={activeQueryField.label}
className="h-4 w-4 rounded-sm border-(--border-muted) text-emerald-500 focus:ring-emerald-500/50"
/>
<span className="truncate text-sm" style={{ color: 'var(--text)' }}>
@@ -800,19 +796,11 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
stroke="currentColor"
aria-hidden="true"
>
{combinedModeLocked ? (
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M16.5 10.5V6.75a4.5 4.5 0 1 0-9 0v3.75m-.75 11.25h10.5a2.25 2.25 0 0 0 2.25-2.25v-6.75a2.25 2.25 0 0 0-2.25-2.25H6.75a2.25 2.25 0 0 0-2.25 2.25v6.75a2.25 2.25 0 0 0 2.25 2.25Z"
/>
) : (
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
/>
)}
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
/>
</svg>
</div>
</div>
@@ -32,7 +32,6 @@ interface SearchSectionProps {
onContentTypeChange?: (type: ContentType) => void;
allowedContentTypes?: ContentType[];
combinedMode?: boolean;
combinedModeLocked?: boolean;
onCombinedModeChange?: (enabled: boolean) => void;
activeQueryField?: MetadataSearchField | null;
searchMode: SearchMode;
@@ -65,7 +64,6 @@ export const SearchSection = ({
onContentTypeChange,
allowedContentTypes,
combinedMode,
combinedModeLocked,
onCombinedModeChange,
activeQueryField,
searchMode,
@@ -107,7 +105,6 @@ export const SearchSection = ({
onContentTypeChange={onContentTypeChange}
allowedContentTypes={allowedContentTypes}
combinedMode={combinedMode}
combinedModeLocked={combinedModeLocked}
onCombinedModeChange={onCombinedModeChange}
queryTargets={queryTargets}
activeQueryTarget={activeQueryTarget}
@@ -12,13 +12,10 @@ interface UrlSearchBootstrapMountProps {
parsedParams: ParsedUrlSearch;
config: AppConfig;
contentType: ContentType;
combinedMode: boolean;
combinedModeAllowed: boolean;
advancedFilters: AdvancedFilterState;
resolvedMetadataDefaultSort: string;
resolvedMetadataSortOptions: SortOption[];
setContentType: (value: ContentType) => void;
setCombinedMode: (value: boolean) => void;
setSearchInput: (value: string) => void;
setAdvancedFilters: Dispatch<SetStateAction<AdvancedFilterState>>;
setShowAdvanced: (value: boolean) => void;
@@ -35,13 +32,10 @@ export const UrlSearchBootstrapMount = ({
parsedParams,
config,
contentType,
combinedMode,
combinedModeAllowed,
advancedFilters,
resolvedMetadataDefaultSort,
resolvedMetadataSortOptions,
setContentType,
setCombinedMode,
setSearchInput,
setAdvancedFilters,
setShowAdvanced,
@@ -55,19 +49,11 @@ export const UrlSearchBootstrapMount = ({
const parsedSearchMode = config.search_mode || 'universal';
const urlContentTypeOverride =
parsedSearchMode === 'universal' ? parsedParams.contentType : undefined;
const urlForcesCombined =
parsedSearchMode === 'universal' && parsedParams.combinedMode === true && combinedModeAllowed;
if (urlContentTypeOverride && urlContentTypeOverride !== contentType) {
setContentType(urlContentTypeOverride);
}
if (urlForcesCombined && !combinedMode) {
setCombinedMode(true);
} else if (urlContentTypeOverride && combinedMode) {
setCombinedMode(false);
}
if (!parsedParams.hasSearchParams) {
return;
}
@@ -436,7 +436,6 @@ const RejectInlinePanel = ({
Reject request for <span className="opacity-80">{itemTitle || 'Untitled request'}</span>
</p>
<textarea
aria-label="Optional note shown to the user"
value={rejectNote}
onChange={(event) => setRejectNote(event.target.value.slice(0, MAX_ADMIN_NOTE_LENGTH))}
rows={3}
@@ -225,9 +225,6 @@ const getInitialPinnedPreference = (): boolean => {
}
};
const EMPTY_KEYS: string[] = [];
const EMPTY_ITEMS: ActivityItem[] = [];
export const ActivitySidebar = ({
isOpen,
onClose,
@@ -238,8 +235,8 @@ export const ActivitySidebar = ({
onRetry,
onDownloadDismiss,
requestItems,
dismissedItemKeys = EMPTY_KEYS,
historyItems = EMPTY_ITEMS,
dismissedItemKeys = [],
historyItems = [],
historyLoaded = false,
historyHasMore = false,
historyLoading = false,
@@ -67,7 +67,6 @@ export const CardView = ({
};
return (
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
<article
className="book-card space-between animate-pop-up relative flex h-full w-full flex-col transition-shadow duration-300 max-sm:h-[180px] max-sm:flex-row sm:max-w-[292px] sm:flex-col"
style={{
@@ -70,7 +70,6 @@ export const CompactView = ({
};
return (
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
<article
className="book-card animate-pop-up relative flex! h-[180px]! w-full flex-row! transition-shadow duration-300"
style={{
@@ -21,6 +21,7 @@ import {
normalizeUserOverrideSections,
UserOverridesSections,
} from './users';
import type { PerUserSettings } from './users/types';
import { UserAccountCardContent, UserEditActions, UserIdentityHeader } from './users/UserCard';
import { useUserOverridesState } from './users/useUserOverridesState';
@@ -152,7 +153,7 @@ const SelfSettingsModalSession = ({
normalizeUserOverrideSections(context.visibleUserSettingsSections, 'self'),
);
applyUserOverridesContext({
settings: context.user.settings || {},
settings: (context.user.settings || {}) as PerUserSettings,
userOverridableKeys: context.userOverridableKeys || [],
});
setEditPassword('');
@@ -102,7 +102,6 @@ export const NamingTemplateField = ({
value={value}
onChange={(event) => onChange(boundField.key, event.target.value)}
placeholder={boundField.placeholder}
aria-label={boundField.placeholder || 'Naming Template'}
maxLength={boundField.maxLength}
disabled={fieldDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
@@ -19,7 +19,6 @@ export const NumberField = ({ field, value, onChange, disabled }: NumberFieldPro
min={field.min}
max={field.max}
step={field.step ?? 1}
aria-label={field.label || 'Number field'}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
/>
@@ -21,7 +21,6 @@ export const PasswordField = ({ field, value, onChange, disabled }: PasswordFiel
value={value}
onChange={(e) => onChange(e.target.value)}
placeholder={field.placeholder}
aria-label={field.label || field.placeholder || 'Password field'}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 pr-10 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
/>
@@ -277,7 +277,6 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
checked={Boolean(cellValue)}
onChange={(e) => updateCell(rowIndex, col.key, e.target.checked)}
disabled={isDisabled}
aria-label={`${col.label || col.key} row ${rowIndex + 1}`}
className="h-4 w-4 rounded border-gray-300 text-sky-600 focus:ring-sky-500 disabled:cursor-not-allowed disabled:opacity-60"
/>
</div>
@@ -362,7 +361,6 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
value={toPrimitiveString(cellValue)}
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
placeholder={col.placeholder}
aria-label={`${col.label || col.key} row ${rowIndex + 1}`}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
/>
@@ -143,7 +143,6 @@ export const TagListField = ({
type="text"
value={draft}
onChange={(e) => setDraft(e.target.value)}
aria-label={field.label || field.placeholder || 'Add item'}
onKeyDown={(e) => {
if (e.key === 'Enter') {
e.preventDefault();
@@ -17,7 +17,6 @@ export const TextField = ({ field, value, onChange, disabled }: TextFieldProps)
value={value ?? ''}
onChange={(e) => onChange(e.target.value)}
placeholder={field.placeholder}
aria-label={field.label || field.placeholder || 'Text field'}
maxLength={field.maxLength}
disabled={isDisabled}
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
@@ -139,11 +139,9 @@ function formatUserOverrideValue(value: unknown): string {
}
}
const EMPTY_DETAILS: Array<{ userId: number; username: string; value: unknown }> = [];
const UserOverriddenBadge = ({
count,
details = EMPTY_DETAILS,
details = [],
}: {
count: number;
details?: Array<{ userId: number; username: string; value: unknown }>;
@@ -60,8 +60,6 @@ const modeDescriptions: Record<RequestPolicyMode, string> = {
blocked: 'Downloads and requests are blocked.',
};
const EMPTY_BASE_RULES: RequestPolicyRuleRow[] = [];
export const RequestPolicyGrid = ({
defaultModes,
onDefaultModeChange,
@@ -69,7 +67,7 @@ export const RequestPolicyGrid = ({
defaultModeOverrides,
defaultModeDisabled,
explicitRules,
baseRules = EMPTY_BASE_RULES,
baseRules = [],
onExplicitRulesChange,
sourceCapabilities,
rulesDisabled = false,

Some files were not shown because too many files have changed in this diff Show More