mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:05:20 +01:00
Compare commits
60
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 CaliBrain
|
9b1d4322b7 | ||
 dependabot[bot]
|
d7a21ea248 | ||
|
dependabot[bot]
|
c487c6c483 | ||
|
CaliBrain
|
404e8cc5c5 | ||
 Alex SchittkoandCaliBrain
|
40d6a179b7 | ||
|
CaliBrain
|
02a45c1a12 | ||
|
dependabot[bot]
|
4c78a2f3a0 | ||
|
Alex Schittko
|
8b4230ee9f | ||
|
dependabot[bot]
|
b57925c39d | ||
|
dependabot[bot]
|
b6efed95ab | ||
|
dependabot[bot]
|
44a261e857 | ||
|
dependabot[bot]
|
14bd50bfd1 | ||
|
dependabot[bot]andCaliBrain
|
4f123977ff | ||
|
dependabot[bot]
|
e3b05d6030 | ||
|
dependabot[bot]
|
995d8d6631 | ||
|
dependabot[bot]
|
9a27334d9c | ||
|
dependabot[bot]
|
76c55a488c | ||
|
dependabot[bot]
|
08d0a04cf2 | ||
|
CaliBrain
|
36f9a52987 | ||
|
CaliBrain
|
1536b91a46 | ||
|
CaliBrain
|
fea63269e3 | ||
|
CaliBrain
|
7886ba7038 | ||
|
CaliBrain
|
1c19326bd0 | ||
|
CaliBrain
|
2f70ed36e4 | ||
|
dependabot[bot]
|
8d54cff383 | ||
|
dependabot[bot]andCaliBrain
|
63fa6213da | ||
|
CaliBrain
|
61b3eaf351 | ||
|
dependabot[bot]
|
7158b79146 | ||
|
dependabot[bot]
|
3c1b79989a | ||
|
dependabot[bot]
|
e41c1d4e8e | ||
  Hrachya ShaginyanandZo Bot
|
ec7e3482bb | ||
|
dependabot[bot]
|
05cd384efc | ||
|
dependabot[bot]
|
880cc4a299 | ||
|
CaliBrain
|
e9668371f7 | ||
|
CaliBrain
|
d55e42fbc6 | ||
|
dependabot[bot]
|
ade4878cfd | ||
 
|
aba1a68dda | ||
 bullitt168
|
6231678c28 | ||
|
dependabot[bot]
|
b5acbc1209 | ||
|
dependabot[bot]
|
b0df608a53 | ||
|
dependabot[bot]
|
68e4e6120e | ||
|
CaliBrain
|
c64c2d374a | ||
|
CaliBrain
|
75c0b0c33a | ||
|
dependabot[bot]
|
d695b636f0 | ||
|
dependabot[bot]
|
bce5ea0641 | ||
|
spindrift
|
0633cfde87 | ||
|
dependabot[bot]
|
b2131fa940 | ||
|
dependabot[bot]andCaliBrain
|
7e8a0248a0 | ||
|
CaliBrain
|
b914f77748 | ||
  
|
5f3a94d3b2 | ||
|
dependabot[bot]andCaliBrain
|
5d5b8d4bef | ||
|
dependabot[bot]andCaliBrain
|
8c86cdc1dd | ||
|
dependabot[bot]andCaliBrain
|
2e25d2d8db | ||
|
dependabot[bot]andCaliBrain
|
709d3828d9 | ||
|
dependabot[bot]
|
d808c362ea | ||
|
dependabot[bot]
|
ad5686a23a | ||
 Alex
|
3d18320096 | ||
|
Alex
|
3ac8d9f681 | ||
|
Alex
|
2f9b42f428 | ||
|
Alex
|
5a53d5910d |
@@ -42,7 +42,15 @@ updates:
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
docker-base-image-digests:
|
||||
# Exclude python from the group on purpose. Dependabot's Docker
|
||||
# pre-release filter is bypassed for *grouped* updates
|
||||
# (dependabot-core#9496), so a grouped python update proposes pre-release
|
||||
# tags like python:3.15.0b2 as if they were a normal stable minor bump.
|
||||
# Updated individually, python is filtered correctly: alpha/beta/rc tags
|
||||
# are skipped and only stable releases (e.g. 3.15.0 once final) are
|
||||
# proposed. node + uv stay grouped into a single digest PR.
|
||||
patterns: ["*"]
|
||||
exclude-patterns: ["python"]
|
||||
ignore:
|
||||
# Node.js: block major-version bumps so dependabot never proposes
|
||||
# moving from one LTS line to a non-LTS "Current" release (e.g. 24 -> 25).
|
||||
|
||||
@@ -67,10 +67,10 @@ jobs:
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -78,7 +78,13 @@ jobs:
|
||||
|
||||
- name: Extract metadata for ${{ matrix.target }} image
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
env:
|
||||
# Annotate both the per-platform manifests and the multi-arch image
|
||||
# index. The index level is what manifest-list consumers (Renovate's
|
||||
# minimumReleaseAge soak check, provenance/SBOM tooling) read for the
|
||||
# standard org.opencontainers.image.* annotations, including `created`.
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
|
||||
tags: |
|
||||
@@ -90,11 +96,11 @@ jobs:
|
||||
type=ref,event=tag
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
|
||||
- name: Build and push ${{ matrix.target }} Docker image
|
||||
id: push
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
@@ -105,10 +111,11 @@ jobs:
|
||||
RELEASE_VERSION=${{ github.ref_name }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
annotations: ${{ steps.meta.outputs.annotations }}
|
||||
|
||||
- name: Generate artifact attestation for ${{ matrix.target }} image
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
|
||||
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
@@ -127,14 +134,14 @@ jobs:
|
||||
LEGACY_NAME: calibre-web-automated-book-downloader
|
||||
steps:
|
||||
- name: Log in to registry
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
|
||||
- name: Create legacy aliases
|
||||
run: |
|
||||
|
||||
+12
-12
@@ -13,10 +13,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -39,10 +39,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -59,10 +59,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -78,13 +78,13 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
|
||||
- name: Build shelfmark-lite image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
with:
|
||||
context: .
|
||||
target: shelfmark-lite
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
|
||||
@@ -22,17 +22,17 @@ jobs:
|
||||
language: [python, javascript-typescript]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
|
||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
|
||||
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3
|
||||
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
name: E2E Platform
|
||||
|
||||
# Hermetic end-to-end matrix: boots the app under test against mock
|
||||
# Anna's Archive / Cloudflare / bypasser / DNS / proxy / Tor / real torrent
|
||||
# clients and runs the cluster suite under each config profile.
|
||||
#
|
||||
# On a PR that touches relevant code, this runs a fast core subset *and* the heavy
|
||||
# `full` profile (real Chrome solving Cloudflare + DoH + real qBittorrent). The
|
||||
# `e2e-required` job aggregates them into ONE status check — make that check a
|
||||
# required status check in branch protection to block merges on any e2e failure
|
||||
# (see tests/e2e/platform/README.md "Gating PRs").
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: "0 4 * * *" # nightly full matrix
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-platform-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Detect whether anything that affects the e2e platform changed. This lets the
|
||||
# required check always report (never stuck "pending") while only spending CI on
|
||||
# PRs that can actually break the e2e stack.
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
relevant: ${{ steps.filter.outputs.relevant }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: dorny/paths-filter@v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
relevant:
|
||||
- 'shelfmark/**'
|
||||
- 'entrypoint.sh'
|
||||
- 'tor.sh'
|
||||
- 'Dockerfile'
|
||||
- 'tests/e2e/platform/**'
|
||||
- '.github/workflows/e2e-platform.yml'
|
||||
|
||||
select-profiles:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
profiles: ${{ steps.pick.outputs.profiles }}
|
||||
steps:
|
||||
- id: pick
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo 'profiles=["baseline","bypasser-external","dns-blocked"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'profiles=["baseline","bypasser-external","bypasser-disabled","dns-manual","dns-blocked","dns-doh","proxy-http","proxy-socks","tor","client-transmission","client-deluge","client-qbittorrent-delayed"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
e2e:
|
||||
needs: select-profiles
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }}
|
||||
name: e2e (${{ matrix.profile }})
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: make install-python-dev
|
||||
- name: Run e2e platform (${{ matrix.profile }})
|
||||
run: tests/e2e/platform/run-e2e.sh env/${{ matrix.profile }}.env
|
||||
- name: Dump shelfmark logs on failure
|
||||
if: failure()
|
||||
run: cat tests/e2e/platform/.state/shelfmark.${{ matrix.profile }}.log || true
|
||||
|
||||
# Heavy "everything real" job: real Chrome internal bypasser solving Cloudflare +
|
||||
# DoH + real qBittorrent webseed download. Runs on relevant PRs and nightly.
|
||||
e2e-full:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
name: e2e (full — real Chrome + qBittorrent)
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: make install-python-dev
|
||||
- name: Run full pipeline
|
||||
run: tests/e2e/platform/run-e2e.sh env/full.env
|
||||
- name: Dump logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
cat tests/e2e/platform/.state/shelfmark.full.log || true
|
||||
docker logs e2e-qbittorrent || true
|
||||
|
||||
# Single aggregated gate. ALWAYS runs (so a required check never hangs "pending"
|
||||
# on unrelated PRs) and FAILS if any e2e job failed/was cancelled. Make THIS the
|
||||
# required status check in branch protection.
|
||||
e2e-required:
|
||||
name: e2e required
|
||||
needs: [e2e, e2e-full]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Aggregate e2e results
|
||||
run: |
|
||||
matrix='${{ needs.e2e.result }}'
|
||||
full='${{ needs.e2e-full.result }}'
|
||||
echo "e2e matrix=$matrix, e2e-full=$full"
|
||||
# success or skipped (unrelated PR) is OK; failure/cancelled blocks.
|
||||
for r in "$matrix" "$full"; do
|
||||
if [ "$r" = "failure" ] || [ "$r" = "cancelled" ]; then
|
||||
echo "::error::An e2e platform job did not pass — blocking."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "All e2e platform jobs passed (or were skipped as not relevant)."
|
||||
@@ -166,6 +166,10 @@ ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# ...but the e2e platform test profiles live in an env/ dir and must be tracked
|
||||
!tests/e2e/platform/env/
|
||||
!tests/e2e/platform/env/*.env
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
+49
-11
@@ -4,7 +4,7 @@ ARG BUILDPLATFORM
|
||||
ARG BUILDARCH
|
||||
|
||||
# Frontend build stage.
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f AS frontend-builder
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS frontend-builder
|
||||
|
||||
# Helpful debug output to see what platforms BuildKit thinks it's using
|
||||
RUN echo "BUILDPLATFORM=$BUILDPLATFORM BUILDARCH=$BUILDARCH TARGETPLATFORM=$TARGETPLATFORM TARGETARCH=$TARGETARCH"
|
||||
@@ -25,7 +25,7 @@ COPY src/frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14-slim@sha256:1697e8e8d39bf168e177ac6b5fdab6df86d81cfc24dae17dfb96cfc3ef76b4dd AS base
|
||||
FROM python:3.14.6-slim@sha256:b877e50bd90de10af8d82c57a022fc2e0dc731c5320d762a27986facfc3355c1 AS base
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 /uv /uvx /bin/
|
||||
|
||||
@@ -72,7 +72,12 @@ RUN apt-get update && \
|
||||
# --- Tor support (activated via USING_TOR=true) ---
|
||||
tor \
|
||||
supervisor \
|
||||
iptables && \
|
||||
iptables \
|
||||
# --- WireGuard support (activated via USING_WIREGUARD=true) ---
|
||||
wireguard-tools \
|
||||
iproute2 \
|
||||
procps \
|
||||
ca-certificates && \
|
||||
# Configure iptables alternatives for tor.sh compatibility
|
||||
update-alternatives --set iptables /usr/sbin/iptables-legacy && \
|
||||
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
|
||||
@@ -131,7 +136,7 @@ RUN mkdir -p \
|
||||
ln -s /tmp/shelfmark/seleniumbase/archived_files /app/archived_files && \
|
||||
chown -R 1000:1000 /config /books /home/shelfmark /tmp/shelfmark /var/log/shelfmark && \
|
||||
chmod -R a+rX /app && \
|
||||
chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
|
||||
chmod +x /app/entrypoint.sh /app/tor.sh /app/wireguard.sh /app/genDebug.sh
|
||||
|
||||
# Expose the application port
|
||||
EXPOSE ${FLASK_PORT}
|
||||
@@ -147,21 +152,39 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
||||
|
||||
FROM base AS shelfmark
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
# --- Chromium (PINNED to 149.0.7827.196) ---
|
||||
# Debian's chromium 150.0.7871.46-1~deb13u1 security update (trixie-security,
|
||||
# 2026-07-05) no longer opens the DevTools remote-debugging TCP port at all
|
||||
# (no listener, no DevToolsActivePort file, even with a custom --user-data-dir;
|
||||
# the RemoteDebuggingAllowed policy does not restore it). The SeleniumBase
|
||||
# Pure-CDP driver connects through that port (/json/version), so with 150 every
|
||||
# internal bypass dies with "Pure CDP browser startup failed" and all
|
||||
# CF-gated downloads fail. Install the last working version from
|
||||
# snapshot.debian.org until the bypasser can talk to Chromium >= 150 (e.g.
|
||||
# pipe-based DevTools / UC mode) or seleniumbase ships a fix.
|
||||
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
|
||||
# This flag is set in internal_bypasser.py _get_browser_args()
|
||||
ARG CHROMIUM_VERSION=149.0.7827.196-1~deb13u1
|
||||
ARG CHROMIUM_SNAPSHOT=20260704T000000Z
|
||||
|
||||
RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/${CHROMIUM_SNAPSHOT}/ trixie-security main" \
|
||||
> /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
|
||||
apt-get update -o Acquire::Retries=5 && \
|
||||
apt-get install -y --no-install-recommends -o Acquire::Retries=5 \
|
||||
# For dumb display
|
||||
xvfb \
|
||||
# For screen recording
|
||||
ffmpeg \
|
||||
# --- Chromium (unpinned - uses latest from Debian repos) ---
|
||||
# Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
|
||||
# This flag is set in internal_bypasser.py _get_browser_args()
|
||||
chromium \
|
||||
chromium-common \
|
||||
chromium=${CHROMIUM_VERSION} \
|
||||
chromium-common=${CHROMIUM_VERSION} \
|
||||
# For tkinter (pyautogui)
|
||||
python3-tk \
|
||||
# For RAR extraction
|
||||
unrar-free && \
|
||||
# Keep apt from "upgrading" chromium past the pin inside derived images
|
||||
printf 'Package: chromium chromium-common\nPin: version %s\nPin-Priority: 1001\n' "${CHROMIUM_VERSION}" \
|
||||
> /etc/apt/preferences.d/chromium-pin && \
|
||||
rm /etc/apt/sources.list.d/chromium-pin-snapshot.list && \
|
||||
# Create symlink so rarfile library can find unrar
|
||||
ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
|
||||
# Cleanup APT cache
|
||||
@@ -173,6 +196,21 @@ RUN apt-get update && \
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --locked --no-default-groups --extra browser
|
||||
|
||||
# Deterministically resolve the Xlib namespace collision.
|
||||
# pyautogui/mouseinfo pull the stale `python3-xlib` (0.15, 2014), while the
|
||||
# `--extra browser` set pulls `python-xlib` (0.33). Both packages install into
|
||||
# the same top-level `Xlib/` namespace, so whichever lands last wins. When the
|
||||
# 2014 build wins, `Xlib.X` is missing `FamilyServerInterpreted`, which the
|
||||
# SeleniumBase Pure-CDP driver requires at browser startup -> every bypass fails
|
||||
# with "module 'Xlib.X' has no attribute 'FamilyServerInterpreted'" and no
|
||||
# Cloudflare/DDoS-Guard protected download can complete. Drop the stale package
|
||||
# and force python-xlib 0.33 to own the namespace. pyautogui runs fine against
|
||||
# 0.33 (superset API).
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip uninstall --python /app/.venv/bin/python python3-xlib && \
|
||||
uv pip install --python /app/.venv/bin/python --reinstall python-xlib==0.33 && \
|
||||
/app/.venv/bin/python -c "import Xlib.X; assert hasattr(Xlib.X, 'FamilyServerInterpreted'), 'Xlib.X.FamilyServerInterpreted missing after fix'; print('Xlib namespace OK:', Xlib.__version__)"
|
||||
|
||||
# uv is only needed while building the image.
|
||||
RUN rm -f /usr/bin/uv /usr/bin/uvx
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov checks fix
|
||||
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov e2e-platform e2e-platform-profile e2e-platform-matrix e2e-platform-full e2e-platform-build checks fix
|
||||
|
||||
# Frontend directory
|
||||
FRONTEND_DIR := src/frontend
|
||||
@@ -38,6 +38,10 @@ help:
|
||||
@echo " python-checks - Run all Python static analysis checks"
|
||||
@echo " python-test - Run unit tests"
|
||||
@echo " python-test-cov - Run unit tests with coverage report"
|
||||
@echo " e2e-platform - Run e2e docker platform (baseline profile)"
|
||||
@echo " e2e-platform-profile PROFILE=<name> - Run e2e platform for one profile"
|
||||
@echo " e2e-platform-matrix - Run e2e platform across all config profiles"
|
||||
@echo " e2e-platform-full - Run heavy 'full' profile (real Chrome bypasser + DoH + real qBittorrent)"
|
||||
@echo " clean - Remove node_modules and build artifacts"
|
||||
@echo ""
|
||||
@echo "Backend (Docker):"
|
||||
@@ -127,6 +131,30 @@ python-test-cov:
|
||||
@echo "Running tests with coverage..."
|
||||
uv run pytest tests/ -x --tb=short -m "not integration and not e2e" --cov --cov-report=term-missing
|
||||
|
||||
# E2E docker platform: hermetic stack (mock AA/Cloudflare/bypasser/DNS/proxy/Tor)
|
||||
# exercised across config profiles. See tests/e2e/platform/README.md.
|
||||
E2E_PLATFORM_DIR := tests/e2e/platform
|
||||
|
||||
e2e-platform:
|
||||
@echo "Running e2e platform (baseline profile)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/baseline.env
|
||||
|
||||
e2e-platform-profile:
|
||||
@echo "Running e2e platform (profile=$(PROFILE))..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/$(PROFILE).env
|
||||
|
||||
e2e-platform-matrix:
|
||||
@echo "Running e2e platform matrix (all profiles)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-matrix.sh
|
||||
|
||||
e2e-platform-build:
|
||||
@echo "Pre-building e2e platform images once (reused across profiles)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./build-images.sh
|
||||
|
||||
e2e-platform-full:
|
||||
@echo "Running e2e platform FULL profile (real Chrome bypasser + DoH + real qBittorrent)..."
|
||||
cd $(E2E_PLATFORM_DIR) && ./run-e2e.sh env/full.env
|
||||
|
||||
# Frontend linting
|
||||
frontend-lint:
|
||||
@echo "Running Oxlint..."
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# Routes all traffic through a WireGuard tunnel - requires root startup.
|
||||
#
|
||||
# Mount your wg-quick config at /config/wg0.conf (read-only is fine). All
|
||||
# non-LAN egress is forced through the tunnel by an iptables kill-switch, so if
|
||||
# the tunnel drops, external traffic fails closed. LAN ranges (WebUI + internal
|
||||
# download clients like Prowlarr / qBittorrent) stay reachable off-tunnel.
|
||||
services:
|
||||
shelfmark-wireguard:
|
||||
image: ghcr.io/calibrain/shelfmark:latest
|
||||
environment:
|
||||
FLASK_PORT: 8084
|
||||
# Quoted so it is passed as the literal string "true": entrypoint.sh compares
|
||||
# $USING_WIREGUARD against "true", and some Compose implementations stringify
|
||||
# a bare YAML boolean as "True", which would silently NOT enable WireGuard.
|
||||
USING_WIREGUARD: "true"
|
||||
# Path to the mounted wg-quick config (default shown).
|
||||
WIREGUARD_CONFIG: /config/wg0.conf
|
||||
# CIDRs kept OFF the tunnel so the WebUI and internal clients stay reachable.
|
||||
LAN_NETWORK: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
# WireGuard needs the module/kernel routing; NET_ADMIN covers wg-quick.
|
||||
sysctls:
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
# Disable IPv6 in the container so the kill-switch can guarantee no IPv6
|
||||
# leak path on kernels/containers without a usable ip6tables. wireguard.sh
|
||||
# fails closed if IPv6 is neither kill-switched nor disabled. If your host
|
||||
# DOES have a working ip6tables you may omit these (an ip6tables kill-switch
|
||||
# is installed instead); or set WIREGUARD_ALLOW_IPV6_LEAK=true only if the
|
||||
# container genuinely has no IPv6 connectivity.
|
||||
- net.ipv6.conf.all.disable_ipv6=1
|
||||
- net.ipv6.conf.default.disable_ipv6=1
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Default destination for book downloads
|
||||
- /path/to/config:/config # App configuration (put wg0.conf here)
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -0,0 +1,25 @@
|
||||
# Local development - WireGuard variant
|
||||
services:
|
||||
shelfmark-wireguard-dev:
|
||||
extends:
|
||||
file: ./compose/docker-compose.wireguard.yml
|
||||
service: shelfmark-wireguard
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
target: shelfmark
|
||||
environment:
|
||||
# Quoted so they are passed as the literal string "true" (entrypoint.sh and
|
||||
# the app compare against "true"); a bare YAML boolean can be stringified as
|
||||
# "True" by some Compose variants, silently disabling the feature.
|
||||
DEBUG: "true"
|
||||
USING_WIREGUARD: "true"
|
||||
WIREGUARD_CONFIG: /config/wg0.conf
|
||||
volumes:
|
||||
- ./.local/config:/config
|
||||
- ./.local/books:/books
|
||||
- ./.local/log:/var/log/shelfmark
|
||||
- ./.local/tmp:/tmp/shelfmark
|
||||
# Place your wg-quick config at ./.local/config/wg0.conf
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -91,6 +91,10 @@ Example:
|
||||
- Shelfmark can see the same files at `/downloads/books/...`
|
||||
- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
|
||||
|
||||
If the files are copied or synced into Shelfmark on a delay, increase **Completed Path Wait (seconds)**
|
||||
in Settings -> Advanced. The default is 60 seconds; seedbox or remote-sync setups may need a value
|
||||
longer than the sync interval.
|
||||
|
||||
## File Processing Options
|
||||
|
||||
### Transfer Method (Torrent / Usenet Only)
|
||||
|
||||
@@ -7,6 +7,7 @@ This document lists all configuration options that can be set via environment va
|
||||
## Table of Contents
|
||||
|
||||
- [Bootstrap Configuration](#bootstrap-configuration)
|
||||
- [Egress / VPN Routing](#egress--vpn-routing)
|
||||
- [General](#general)
|
||||
- [Search Mode](#search-mode)
|
||||
- [Downloads](#downloads)
|
||||
@@ -145,6 +146,98 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme
|
||||
|
||||
</details>
|
||||
|
||||
## Egress / VPN Routing
|
||||
|
||||
These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.
|
||||
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `USING_WIREGUARD` | Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR. | boolean | `false` |
|
||||
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick configuration file. | string (path) | `/config/wg0.conf` |
|
||||
| `WIREGUARD_INTERFACE` | WireGuard interface name brought up by wg-quick. | string | `wg0` |
|
||||
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable. | string (comma-separated) | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
|
||||
| `WIREGUARD_ENFORCE_DNS` | Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable. | boolean | `true` |
|
||||
| `WIREGUARD_DNS` | Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead. | string (comma-separated) | `unset (uses config DNS = line)` |
|
||||
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface. | boolean | `true` |
|
||||
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel. | boolean | `false` |
|
||||
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK). | boolean | `false` |
|
||||
| `WIREGUARD_STALE_AFTER` | Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel. | number | `180` |
|
||||
|
||||
<details>
|
||||
<summary>Detailed descriptions</summary>
|
||||
|
||||
#### `USING_WIREGUARD`
|
||||
|
||||
Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `WIREGUARD_CONFIG`
|
||||
|
||||
Path to the mounted wg-quick configuration file.
|
||||
|
||||
- **Type:** string (path)
|
||||
- **Default:** `/config/wg0.conf`
|
||||
|
||||
#### `WIREGUARD_INTERFACE`
|
||||
|
||||
WireGuard interface name brought up by wg-quick.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** `wg0`
|
||||
|
||||
#### `LAN_NETWORK`
|
||||
|
||||
Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`
|
||||
|
||||
#### `WIREGUARD_ENFORCE_DNS`
|
||||
|
||||
Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `true`
|
||||
|
||||
#### `WIREGUARD_DNS`
|
||||
|
||||
Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `unset (uses config DNS = line)`
|
||||
|
||||
#### `WIREGUARD_DISABLE_IPV6`
|
||||
|
||||
Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `true`
|
||||
|
||||
#### `WIREGUARD_ALLOW_IPV6_LEAK`
|
||||
|
||||
Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL`
|
||||
|
||||
When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `WIREGUARD_STALE_AFTER`
|
||||
|
||||
Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `180`
|
||||
|
||||
</details>
|
||||
|
||||
## General
|
||||
|
||||
| Variable | Description | Type | Default |
|
||||
@@ -223,6 +316,7 @@ Default language filter for searches.
|
||||
| `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` |
|
||||
| `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` |
|
||||
| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` |
|
||||
| `FORCE_COMBINED_SEARCH` | Force combined search whenever it's available. Locks the combined toggle on. | boolean | `false` |
|
||||
| `METADATA_PROVIDER` | Choose which metadata provider to use for book searches. | string (choice) | `openlibrary` |
|
||||
| `METADATA_PROVIDER_AUDIOBOOK` | Metadata provider for audiobook searches. Uses the book provider if not set. | string (choice) | _empty string_ |
|
||||
| `METADATA_PROVIDER_COMBINED` | Metadata provider for combined mode searches. Uses the book provider if not set. | string (choice) | _empty string_ |
|
||||
@@ -270,6 +364,15 @@ Show the option to search for and download both a book and audiobook together.
|
||||
- **Type:** boolean
|
||||
- **Default:** `true`
|
||||
|
||||
#### `FORCE_COMBINED_SEARCH`
|
||||
|
||||
**Always Use Combined Search**
|
||||
|
||||
Force combined search whenever it's available. Locks the combined toggle on.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `METADATA_PROVIDER`
|
||||
|
||||
**Book Metadata Provider**
|
||||
@@ -948,6 +1051,7 @@ Comma-separated hosts to bypass proxy (e.g., localhost,127.0.0.1,10.*,*.local)
|
||||
| `CUSTOM_SCRIPT` | Path to a script to run after each successful download. Must be executable. | string | _none_ |
|
||||
| `CUSTOM_SCRIPT_PATH_MODE` | Pass the path to the custom script as an absolute path or relative to the destination folder. | string (choice) | `absolute` |
|
||||
| `CUSTOM_SCRIPT_JSON_PAYLOAD` | Send a JSON payload to the script via stdin. Useful for multi-file imports (audiobooks) or richer metadata without relying on path parsing. | boolean | `false` |
|
||||
| `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT` | How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows. | number | `60` |
|
||||
| `COVERS_CACHE_ENABLED` | Cache book covers on the server for faster loading. | boolean | `true` |
|
||||
| `COVERS_CACHE_TTL` | How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork). | number | `0` |
|
||||
| `COVERS_CACHE_MAX_SIZE_MB` | Maximum disk space for cached covers. Oldest images are removed when limit is reached. | number | `500` |
|
||||
@@ -1028,6 +1132,16 @@ Send a JSON payload to the script via stdin. Useful for multi-file imports (audi
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT`
|
||||
|
||||
**Completed Path Wait (seconds)**
|
||||
|
||||
How long to wait after a torrent or usenet client reports completion for the completed file path to become visible to Shelfmark. Increase this for seedbox or remote-sync workflows.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `60`
|
||||
- **Constraints:** min: 0, max: 3600
|
||||
|
||||
#### `COVERS_CACHE_ENABLED`
|
||||
|
||||
**Enable Cover Cache**
|
||||
@@ -1283,7 +1397,7 @@ Delay between requests in seconds to avoid rate limiting (0-10).
|
||||
| `IRC_USE_TLS` | Enable TLS/SSL encryption for the IRC connection. Disable for servers that don't support TLS. | boolean | `true` |
|
||||
| `IRC_CHANNEL` | Channel name without the # prefix | string | _none_ |
|
||||
| `IRC_NICK` | Your IRC nickname (required). Must be unique on the IRC network. | string | _none_ |
|
||||
| `IRC_SEARCH_BOT` | The search bot to query for results | string | _none_ |
|
||||
| `IRC_SEARCH_BOT` | The search bot to address queries to (required). | string | _none_ |
|
||||
| `IRC_CACHE_TTL` | How long to keep cached search results before they expire. | string (choice) | `2592000` |
|
||||
|
||||
<details>
|
||||
@@ -1341,10 +1455,11 @@ Your IRC nickname (required). Must be unique on the IRC network.
|
||||
|
||||
**Search bot**
|
||||
|
||||
The search bot to query for results
|
||||
The search bot to address queries to (required). Searches are sent as "@<bot> <query>". Without it, queries would be posted unaddressed to the channel.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** _none_
|
||||
- **Required:** Yes
|
||||
|
||||
#### `IRC_CACHE_TTL`
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ http://your-server:8084/?q=harry+potter
|
||||
| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
|
||||
| `format` | Filter by file format | `/?format=epub` |
|
||||
| `content` | Filter by content type | `/?content=fiction` |
|
||||
| `content_type` | Select media type (`ebook` or `audiobook`) in Universal mode only | `/?q=dune&content_type=audiobook` |
|
||||
| `content_type` | Select media type (`ebook`, `audiobook`, or `combined`) in Universal mode only | `/?q=dune&content_type=audiobook` |
|
||||
| `sort` | Sort order for results | `/?sort=newest` |
|
||||
|
||||
## Multiple Values
|
||||
@@ -63,6 +63,11 @@ Some parameters support multiple values by repeating the parameter:
|
||||
/?q=dune&content_type=audiobook
|
||||
```
|
||||
|
||||
**Universal search forcing combined (ebook + audiobook):**
|
||||
```
|
||||
/?q=dune&content_type=combined
|
||||
```
|
||||
|
||||
## Search Mode Behavior
|
||||
|
||||
### Direct Mode
|
||||
@@ -74,6 +79,8 @@ When Search Mode is set to Direct, all parameters are used to filter results fro
|
||||
|
||||
`q`, `sort`, and `content_type` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
|
||||
|
||||
`content_type=combined` forces combined mode (search ebook and audiobook providers together), overriding the last-used preference. It is silently ignored if combined mode is unavailable (e.g. the combined selector is disabled in settings, or either content type is blocked by request policy).
|
||||
|
||||
## Notes
|
||||
|
||||
- URL parameters are read once on page load
|
||||
|
||||
+27
-1
@@ -81,6 +81,13 @@ if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Egress modes are mutually exclusive. Check this BEFORE starting either one so
|
||||
# we never run tor.sh and then abort, leaving a half-configured network stack.
|
||||
if [ "$USING_TOR" = "true" ] && [ "$USING_WIREGUARD" = "true" ]; then
|
||||
echo "USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$USING_TOR" = "true" ]; then
|
||||
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
|
||||
echo "USING_TOR=true requires the container to start as root." >&2
|
||||
@@ -90,6 +97,15 @@ if [ "$USING_TOR" = "true" ]; then
|
||||
./tor.sh
|
||||
fi
|
||||
|
||||
if [ "$USING_WIREGUARD" = "true" ]; then
|
||||
if [ "$RUN_AS_NON_ROOT" = "true" ]; then
|
||||
echo "USING_WIREGUARD=true requires the container to start as root." >&2
|
||||
echo "Non-root mode skips the privileged network setup WireGuard depends on." >&2
|
||||
exit 1
|
||||
fi
|
||||
./wireguard.sh
|
||||
fi
|
||||
|
||||
if [ "$FILE_LOGGING_ENABLED" = "true" ]; then
|
||||
start_file_logging "$LOG_FILE"
|
||||
fi
|
||||
@@ -235,7 +251,11 @@ test_write() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! run_as_target_user sh -c 'echo 0123456789_TEST > "$1"' _ "$test_file"; then
|
||||
# This is a probe: a failure here is expected (e.g. a fresh root-owned bind
|
||||
# mount) and is recovered by the caller via change_ownership + re-probe. Hide
|
||||
# the shell's "Permission denied"/"Read-only file system" stderr so a handled
|
||||
# probe miss doesn't masquerade as a real boot failure in the logs.
|
||||
if ! run_as_target_user sh -c 'echo 0123456789_TEST 2>/dev/null > "$1"' _ "$test_file"; then
|
||||
echo "Failed to write test file in $folder as $USERNAME"
|
||||
return 1
|
||||
fi
|
||||
@@ -448,6 +468,12 @@ else
|
||||
if [ $config_ok -ne 0 ]; then
|
||||
fail_unwritable_config_dir "$CONFIG_PATH"
|
||||
fi
|
||||
|
||||
# The ingest/destination library (default /books) is user data and may be a
|
||||
# bind mount owned by another uid; downloads fail with "Destination not
|
||||
# writable" if the runtime user can't write there. Fix the top-level dir only
|
||||
# (root mode) so we don't recursively chown a potentially huge library.
|
||||
make_writable "${INGEST_DIR:-/books}" root
|
||||
fi
|
||||
|
||||
# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
|
||||
|
||||
+4
-4
@@ -22,25 +22,25 @@ dependencies = [
|
||||
"qbittorrent-api",
|
||||
"transmission-rpc",
|
||||
"authlib>=1.7.2,<1.8",
|
||||
"apprise>=1.10.0",
|
||||
"apprise>=1.12.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
browser = [
|
||||
"pyvirtualdisplay",
|
||||
"pyautogui",
|
||||
"seleniumbase==4.48.4",
|
||||
"seleniumbase==4.50.5",
|
||||
"python-xlib",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.39.3",
|
||||
"basedpyright>=1.39.9",
|
||||
"prek",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-xdist>=3.8.0",
|
||||
"ruff==0.15.12",
|
||||
"ruff==0.15.20",
|
||||
"vulture>=2.14",
|
||||
]
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
|
||||
|
||||
> [!NOTE]
|
||||
> This project is in a stable state as of May 2026 but is not under active maintenance.
|
||||
|
||||
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
|
||||
|
||||
Works great alongside the following library tools, with support for automatic imports:
|
||||
@@ -103,6 +106,16 @@ Environment variables work for initial setup and Docker deployments. They serve
|
||||
| `PUID` / `PGID` | Runtime user/group for the default root-startup flow (also supports legacy `UID`/`GID`) | `1000` / `1000` |
|
||||
| `SEARCH_MODE` | `direct` or `universal` | `universal` |
|
||||
| `USING_TOR` | Enable Tor routing (requires root startup) | `false` |
|
||||
| `USING_WIREGUARD` | Enable WireGuard VPN egress with kill-switch (requires root startup) | `false` |
|
||||
| `WIREGUARD_CONFIG` | Path to the mounted wg-quick config | `/config/wg0.conf` |
|
||||
| `WIREGUARD_INTERFACE` | WireGuard interface name | `wg0` |
|
||||
| `LAN_NETWORK` | Comma-separated CIDRs kept off the tunnel so the WebUI / internal clients stay reachable | `127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` |
|
||||
| `WIREGUARD_ENFORCE_DNS` | Pin the resolver (via `WIREGUARD_DNS`, else the config's `DNS =`) so DNS can't silently fall back to an off-tunnel path. Designed for a trusted LAN resolver kept reachable via `LAN_NETWORK` (query leaves over the LAN; download still egresses via the tunnel) — it does **not** force queries through the tunnel. Docker's embedded resolver (`127.0.0.11`) is preserved when present so container-name resolution keeps working; pin its upstream via the container's `dns:` list. Fails closed if no resolver is available or `/etc/resolv.conf` is not writable. | `true` |
|
||||
| `WIREGUARD_DNS` | Explicit resolver(s) to pin (comma/space separated). Use when the VPN's pushed DNS filters domains you need; point at a resolver reachable via the tunnel or an allowed LAN resolver. | _(unset; uses config `DNS =`)_ |
|
||||
| `WIREGUARD_DISABLE_IPV6` | Strip IPv6 from the tunnel config (many container kernels lack the ip6tables `raw` table wg-quick needs) and remove IPv6 as a leak surface. | `true` |
|
||||
| `WIREGUARD_ALLOW_IPV6_LEAK` | Escape hatch: continue even when an IPv6 kill-switch can't be installed AND IPv6 can't be disabled. Only set if the container has no IPv6 connectivity. | `false` |
|
||||
| `WIREGUARD_ALLOW_WEBUI_OFFTUNNEL` | Opt-in off-tunnel WebUI reachability. Default (`false`) keeps the kill-switch strictly fail-closed: the only off-tunnel egress is loopback, the tunnel device and the LAN allowlist. Set `true` only if a **non-LAN** client (e.g. a public reverse proxy on another segment) must reach the WebUI; it permits app-server **replies** (`--sport FLASK_PORT`, conntrack REPLY) off-tunnel — server replies only, never client-initiated egress. LAN clients never need it (covered by `LAN_NETWORK`). | `false` |
|
||||
| `WIREGUARD_STALE_AFTER` | Seconds since the last handshake before the healthcheck bounces the tunnel. | `180` |
|
||||
|
||||
See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
|
||||
|
||||
@@ -138,6 +151,24 @@ docker compose -f docker-compose.tor.yml up -d
|
||||
- Timezone is auto-detected from Tor exit node
|
||||
- Custom DNS/proxy settings are ignored when Tor is active
|
||||
|
||||
#### WireGuard VPN Routing
|
||||
Optional WireGuard support to route all external egress through a VPN tunnel with a fail-closed kill-switch:
|
||||
```bash
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.wireguard.yml
|
||||
# place your wg-quick config where the compose mounts /config, as wg0.conf
|
||||
docker compose -f docker-compose.wireguard.yml up -d
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Requires root startup
|
||||
- Requires `NET_ADMIN` and `NET_RAW` capabilities
|
||||
- Mount a standard wg-quick config at `WIREGUARD_CONFIG` (default `/config/wg0.conf`)
|
||||
- All non-LAN egress is forced through the tunnel; if the tunnel drops, external traffic **fails closed** while LAN ranges (WebUI, Prowlarr, qBittorrent) stay reachable
|
||||
- IPv4 and IPv6 both fail closed. On kernels without a usable `ip6tables`, disable IPv6 for the container (`sysctls: net.ipv6.conf.all.disable_ipv6=1`, as in the compose example) or the container refuses to start rather than risk an IPv6 leak
|
||||
- A supervised healthcheck bounces the tunnel if the handshake goes stale, and refreshes the endpoint allow rules so a roaming/rotated peer endpoint can reconnect
|
||||
- Mutually exclusive with `USING_TOR`
|
||||
- **DNS trust:** `WIREGUARD_DNS` must be a resolver you trust on a trusted network segment. When it is a LAN resolver (kept reachable off-tunnel by `LAN_NETWORK`), the query to that resolver leaves as plaintext UDP/53 on the LAN — the resolver is responsible for encrypting upstream. Two resolver paths exist: (1) when Docker's embedded resolver (`127.0.0.11`) is present it is **preserved** so container names (Prowlarr, qBittorrent) resolve — you MUST pin its upstream to a trusted resolver via the container's compose `dns:` list, since `WIREGUARD_DNS` cannot repoint the embedded resolver from inside the container; (2) otherwise `WIREGUARD_DNS`/the config `DNS =` line is written to `/etc/resolv.conf`. Setting `WIREGUARD_ENFORCE_DNS=false` is a **foot-gun**: with no embedded resolver present the container then uses its inherited resolver, which forwards to the Docker daemon's upstream **off-tunnel**, leaking your DNS. Leave enforcement on unless you have pinned the resolver another way.
|
||||
|
||||
### Lite
|
||||
A lighter image without the built-in browser automation. Ideal for:
|
||||
|
||||
|
||||
@@ -238,6 +238,113 @@ def _generate_bootstrap_env_docs() -> list[str]:
|
||||
return lines
|
||||
|
||||
|
||||
def _generate_egress_env_docs() -> list[str]:
|
||||
"""Generate documentation for VPN/Tor egress environment variables.
|
||||
|
||||
These are startup-only variables consumed by entrypoint.sh / wireguard.sh
|
||||
(before and outside the settings registry) to select and configure the
|
||||
transparent-egress kill-switch. `USING_TOR` has a registry-backed entry
|
||||
under Network and is cross-referenced rather than repeated here so the two
|
||||
mutually exclusive egress modes are discoverable side by side without
|
||||
emitting a duplicate `#### USING_TOR` anchor.
|
||||
"""
|
||||
egress_vars = [
|
||||
{
|
||||
"name": "USING_WIREGUARD",
|
||||
"description": "Route all traffic through a WireGuard VPN tunnel with a fail-closed iptables kill-switch (non-tunnel egress is dropped). Requires root startup and NET_ADMIN (plus NET_RAW). Mutually exclusive with USING_TOR.",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_CONFIG",
|
||||
"description": "Path to the mounted wg-quick configuration file.",
|
||||
"type": "string (path)",
|
||||
"default": "/config/wg0.conf",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_INTERFACE",
|
||||
"description": "WireGuard interface name brought up by wg-quick.",
|
||||
"type": "string",
|
||||
"default": "wg0",
|
||||
},
|
||||
{
|
||||
"name": "LAN_NETWORK",
|
||||
"description": "Comma-separated CIDRs kept off the tunnel so the WebUI and internal download clients (Prowlarr, qBittorrent) stay reachable.",
|
||||
"type": "string (comma-separated)",
|
||||
"default": "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_ENFORCE_DNS",
|
||||
"description": "Pin the container's resolver so DNS cannot silently fall back to an off-tunnel path. The resolver used is WIREGUARD_DNS if set, else the tunnel config's DNS = line. This does NOT force queries through the tunnel: it is designed for a trusted LAN resolver kept reachable off-tunnel via LAN_NETWORK (the query leaves over the LAN; the resolver encrypts upstream while the download still egresses via the tunnel). Special case: when Docker's embedded resolver (nameserver 127.0.0.11) is present, it is PRESERVED so container-name resolution (e.g. prowlarr, qbittorrent) keeps working, and the embedded resolver's upstream must be pinned via the container's compose dns: list. Fails closed (refuses to start) only when no embedded resolver is present AND no resolver is defined, or /etc/resolv.conf is not writable.",
|
||||
"type": "boolean",
|
||||
"default": "true",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_DNS",
|
||||
"description": "Explicit resolver(s) (comma/space separated) to pin when WIREGUARD_ENFORCE_DNS is true and Docker's embedded resolver is NOT in use. Use when the VPN's pushed DNS filters domains you need; point it at a resolver reachable via the tunnel or an allowed LAN resolver. NOTE: when the embedded resolver (127.0.0.11) is present it is preserved and this value cannot repoint its upstream from inside the container — set the container's compose dns: list to the trusted resolver instead.",
|
||||
"type": "string (comma-separated)",
|
||||
"default": "unset (uses config DNS = line)",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_DISABLE_IPV6",
|
||||
"description": "Strip IPv6 Address/AllowedIPs/DNS from the tunnel config before wg-quick (many container kernels lack the ip6tables raw table wg-quick needs) and remove IPv6 as a leak surface.",
|
||||
"type": "boolean",
|
||||
"default": "true",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_ALLOW_IPV6_LEAK",
|
||||
"description": "Escape hatch: continue startup even when an IPv6 kill-switch cannot be installed AND IPv6 cannot be disabled. Only set when the container has no IPv6 connectivity, as IPv6 egress may otherwise bypass the tunnel.",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_ALLOW_WEBUI_OFFTUNNEL",
|
||||
"description": "When false (default) the kill-switch is strictly fail-closed: the only off-tunnel egress permitted is loopback, the tunnel device and the LAN allowlist. Set true only if a NON-LAN client (e.g. a public reverse proxy on a different segment) must reach the WebUI; it permits app-server REPLY packets (--sport FLASK_PORT, conntrack REPLY) to leave off-tunnel. Server replies only, never client-initiated egress, so it cannot leak outbound browsing/downloads or the real IP for outbound requests, but it is still an off-tunnel path while the tunnel is down, hence opt-in. LAN WebUI clients never need it (covered by LAN_NETWORK).",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "WIREGUARD_STALE_AFTER",
|
||||
"description": "Seconds since the last WireGuard handshake before the healthcheck bounces the tunnel.",
|
||||
"type": "number",
|
||||
"default": "180",
|
||||
},
|
||||
]
|
||||
|
||||
lines = [
|
||||
"## Egress / VPN Routing",
|
||||
"",
|
||||
"These startup-only variables are consumed by `entrypoint.sh` / `wireguard.sh` to select and configure the WireGuard transparent-egress kill-switch. `USING_WIREGUARD` and [`USING_TOR`](#using_tor) (documented under Network) are mutually exclusive; both require root startup.",
|
||||
"",
|
||||
"| Variable | Description | Type | Default |",
|
||||
"|----------|-------------|------|---------|",
|
||||
]
|
||||
|
||||
lines.extend(
|
||||
f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |"
|
||||
for var in egress_vars
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
lines.append("<details>")
|
||||
lines.append("<summary>Detailed descriptions</summary>")
|
||||
lines.append("")
|
||||
|
||||
for var in egress_vars:
|
||||
lines.append(f"#### `{var['name']}`")
|
||||
lines.append("")
|
||||
lines.append(var["description"])
|
||||
lines.append("")
|
||||
lines.append(f"- **Type:** {var['type']}")
|
||||
lines.append(f"- **Default:** `{var['default']}`")
|
||||
lines.append("")
|
||||
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def generate_env_docs() -> str:
|
||||
"""Generate markdown documentation for all environment variables."""
|
||||
# Import settings modules to ensure all settings are registered
|
||||
@@ -282,6 +389,7 @@ def generate_env_docs() -> str:
|
||||
# Generate TOC
|
||||
toc_entries = [
|
||||
"- [Bootstrap Configuration](#bootstrap-configuration)",
|
||||
"- [Egress / VPN Routing](#egress--vpn-routing)",
|
||||
]
|
||||
|
||||
# Ungrouped tabs first
|
||||
@@ -307,6 +415,9 @@ def generate_env_docs() -> str:
|
||||
# Add bootstrap environment variables documentation
|
||||
lines.extend(_generate_bootstrap_env_docs())
|
||||
|
||||
# Add egress / VPN routing (startup-only, shell-driven) documentation
|
||||
lines.extend(_generate_egress_env_docs())
|
||||
|
||||
# Generate documentation for ungrouped tabs
|
||||
for tab in grouped_tabs.get(None, []):
|
||||
lines.extend(_generate_tab_docs(tab))
|
||||
|
||||
@@ -939,7 +939,17 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
|
||||
result_path = (
|
||||
Path(tempfile.gettempdir()) / f"shelfmark-bypass-{os.getpid()}-{time.time_ns()}.json"
|
||||
)
|
||||
payload = {"url": url, "retry": retry, "result_path": str(result_path)}
|
||||
# DNS provider state lives only in the parent's memory (no disk persistence), so the
|
||||
# freshly spawned helper would otherwise pre-resolve AA hostnames against the system
|
||||
# resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
|
||||
# active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
|
||||
# resolver rules.
|
||||
payload = {
|
||||
"url": url,
|
||||
"retry": retry,
|
||||
"result_path": str(result_path),
|
||||
"dns_config": network.get_dns_config(),
|
||||
}
|
||||
env_vars = os.environ.copy()
|
||||
env_vars[_BYPASS_CHILD_ENV] = "1"
|
||||
env_vars = _prepare_child_browser_env(env_vars)
|
||||
@@ -1278,6 +1288,30 @@ def get_bypassed_page(
|
||||
return response_html
|
||||
|
||||
|
||||
def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
|
||||
"""Mirror the parent process's active DNS provider in this helper subprocess.
|
||||
|
||||
DNS state is in-memory only, so a fresh helper defaults to system DNS and would
|
||||
pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
|
||||
that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
|
||||
the same DoH/custom resolver the parent already validated.
|
||||
"""
|
||||
provider = str(dns_config.get("provider") or "").strip().lower()
|
||||
# "auto" means the parent has not rotated off system DNS yet, so the helper's own
|
||||
# default initialization already matches it - nothing to override.
|
||||
if not provider or provider == "auto":
|
||||
return
|
||||
manual_servers = dns_config.get("servers") if provider == "manual" else None
|
||||
try:
|
||||
network.set_dns_provider(
|
||||
provider,
|
||||
manual_servers,
|
||||
use_doh=bool(dns_config.get("doh_enabled")),
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
|
||||
|
||||
|
||||
def _run_child_process() -> int:
|
||||
"""CLI entrypoint used by the Docker helper subprocess."""
|
||||
request = json.loads(sys.stdin.read() or "{}")
|
||||
@@ -1287,6 +1321,10 @@ def _run_child_process() -> int:
|
||||
request.get("retry"), _coerce_positive_int(app_config.MAX_RETRY, 10)
|
||||
)
|
||||
|
||||
dns_config = request.get("dns_config")
|
||||
if isinstance(dns_config, dict):
|
||||
_apply_parent_dns_config(dns_config)
|
||||
|
||||
try:
|
||||
html = get(url, retry=retry)
|
||||
payload = {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
from shelfmark.core.config import config
|
||||
@@ -23,7 +24,7 @@ _BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
|
||||
|
||||
|
||||
def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
|
||||
return f"{base_url}|{username}|{hash(password)}"
|
||||
return f"{base_url}|{username}|{hashlib.sha256(password.encode()).hexdigest()}"
|
||||
|
||||
|
||||
def _get_booklore_select_options(
|
||||
|
||||
@@ -36,6 +36,9 @@ from shelfmark.core.settings_registry import (
|
||||
register_settings,
|
||||
)
|
||||
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
|
||||
|
||||
|
||||
def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate advanced settings before persisting."""
|
||||
@@ -43,6 +46,40 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
timeout_key = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
|
||||
if timeout_key in values:
|
||||
raw_timeout = values.get(timeout_key)
|
||||
if isinstance(raw_timeout, bool):
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Completed Path Wait must be a number of seconds",
|
||||
"values": values,
|
||||
}
|
||||
if raw_timeout is None:
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Completed Path Wait must be a number of seconds",
|
||||
"values": values,
|
||||
}
|
||||
try:
|
||||
timeout_seconds = int(raw_timeout)
|
||||
except TypeError, ValueError:
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Completed Path Wait must be a number of seconds",
|
||||
"values": values,
|
||||
}
|
||||
if timeout_seconds < 0 or timeout_seconds > _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX:
|
||||
return {
|
||||
"error": True,
|
||||
"message": (
|
||||
"Completed Path Wait must be between 0 and "
|
||||
f"{_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX} seconds"
|
||||
),
|
||||
"values": values,
|
||||
}
|
||||
values[timeout_key] = timeout_seconds
|
||||
|
||||
mappings = values.get("PROWLARR_REMOTE_PATH_MAPPINGS")
|
||||
if mappings is None:
|
||||
return {"error": False, "values": values}
|
||||
@@ -453,6 +490,14 @@ def search_mode_settings() -> list[SettingsField]:
|
||||
show_when={"field": "SEARCH_MODE", "value": "universal"},
|
||||
user_overridable=True,
|
||||
),
|
||||
CheckboxField(
|
||||
key="FORCE_COMBINED_SEARCH",
|
||||
label="Always Use Combined Search",
|
||||
description="Force combined search whenever it's available. Locks the combined toggle on.",
|
||||
default=False,
|
||||
show_when={"field": "SEARCH_MODE", "value": "universal"},
|
||||
user_overridable=True,
|
||||
),
|
||||
HeadingField(
|
||||
key="universal_mode_heading",
|
||||
title="Universal Mode Settings",
|
||||
@@ -1440,6 +1485,17 @@ def download_source_settings() -> list[SettingsField]:
|
||||
),
|
||||
default=False,
|
||||
),
|
||||
CheckboxField(
|
||||
key="DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH",
|
||||
label="Detect Language From Distant Path",
|
||||
description=(
|
||||
"When language metadata is missing or unknown, parse the distant path "
|
||||
"(file path shown in search results) for language tags like [BD FR] or [En]. "
|
||||
"Also enables local language filtering so lgli files without AA language "
|
||||
"metadata are not excluded before the distant path can be checked."
|
||||
),
|
||||
default=False,
|
||||
),
|
||||
PasswordField(
|
||||
key="AA_DONATOR_KEY",
|
||||
label="Account Donator Key",
|
||||
@@ -1761,6 +1817,18 @@ def advanced_settings() -> list[SettingsField]:
|
||||
title="Remote Path Mappings",
|
||||
description="Map download client paths to paths inside Shelfmark. Needed when volume mounts differ between containers.",
|
||||
),
|
||||
NumberField(
|
||||
key="DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT",
|
||||
label="Completed Path Wait (seconds)",
|
||||
description=(
|
||||
"How long to wait after a torrent or usenet client reports completion "
|
||||
"for the completed file path to become visible to Shelfmark. Increase "
|
||||
"this for seedbox or remote-sync workflows."
|
||||
),
|
||||
default=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT,
|
||||
min_value=0,
|
||||
max_value=_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX,
|
||||
),
|
||||
TableField(
|
||||
key="PROWLARR_REMOTE_PATH_MAPPINGS",
|
||||
label="Path Mappings",
|
||||
|
||||
@@ -82,6 +82,7 @@ _SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
|
||||
"DEFAULT_RELEASE_SOURCE",
|
||||
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
|
||||
"SHOW_COMBINED_SELECTOR",
|
||||
"FORCE_COMBINED_SEARCH",
|
||||
*_SEARCH_PREFERENCE_PROVIDER_KEYS,
|
||||
}
|
||||
|
||||
@@ -223,6 +224,11 @@ def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | N
|
||||
return value, None
|
||||
return bool(value), None
|
||||
|
||||
if key == "FORCE_COMBINED_SEARCH":
|
||||
if isinstance(value, bool):
|
||||
return value, None
|
||||
return bool(value), None
|
||||
|
||||
return value, None
|
||||
|
||||
|
||||
|
||||
@@ -393,6 +393,41 @@ def _plugin_label(plugin: object, fallback_scheme: str) -> str:
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _apprise_proxy_env() -> dict[str, str]:
|
||||
"""Build proxy env vars from app config so Apprise respects the proxy setting."""
|
||||
import os
|
||||
|
||||
from shelfmark.core.config import config as _cfg
|
||||
|
||||
mode = str(_cfg.get("PROXY_MODE", "") or "").lower()
|
||||
env: dict[str, str] = {}
|
||||
|
||||
if mode == "http":
|
||||
http = str(_cfg.get("HTTP_PROXY", "") or "").strip()
|
||||
https = str(_cfg.get("HTTPS_PROXY", "") or "").strip() or http
|
||||
if http:
|
||||
env["HTTP_PROXY"] = http
|
||||
env["http_proxy"] = http
|
||||
if https:
|
||||
env["HTTPS_PROXY"] = https
|
||||
env["https_proxy"] = https
|
||||
elif mode == "socks5":
|
||||
socks = str(_cfg.get("SOCKS5_PROXY", "") or "").strip()
|
||||
if socks:
|
||||
env["HTTP_PROXY"] = socks
|
||||
env["http_proxy"] = socks
|
||||
env["HTTPS_PROXY"] = socks
|
||||
env["https_proxy"] = socks
|
||||
|
||||
no_proxy = str(_cfg.get("NO_PROXY", "") or "").strip()
|
||||
if no_proxy and env:
|
||||
env["NO_PROXY"] = no_proxy
|
||||
env["no_proxy"] = no_proxy
|
||||
|
||||
# Don't override if the user already set these in the environment directly
|
||||
return {k: v for k, v in env.items() if not os.environ.get(k)}
|
||||
|
||||
|
||||
def _dispatch_to_apprise(
|
||||
urls: Iterable[str],
|
||||
*,
|
||||
@@ -400,6 +435,8 @@ def _dispatch_to_apprise(
|
||||
body: str,
|
||||
notify_type: object,
|
||||
) -> dict[str, Any]:
|
||||
import os
|
||||
|
||||
normalized_urls = _normalize_urls(list(urls))
|
||||
url_schemes = _extract_url_schemes(normalized_urls)
|
||||
if not normalized_urls:
|
||||
@@ -408,6 +445,11 @@ def _dispatch_to_apprise(
|
||||
if apprise is None:
|
||||
return {"success": False, "message": "Apprise is not installed"}
|
||||
|
||||
proxy_env = _apprise_proxy_env()
|
||||
if proxy_env:
|
||||
logger.debug("Applying proxy env for Apprise dispatch: %s", list(proxy_env.keys()))
|
||||
os.environ.update(proxy_env)
|
||||
|
||||
valid_urls = 0
|
||||
invalid_urls = 0
|
||||
delivered_urls = 0
|
||||
|
||||
@@ -52,6 +52,13 @@ def normalize_http_url(
|
||||
if scheme:
|
||||
normalized = f"{scheme}://{normalized}"
|
||||
|
||||
# Strip query string and fragment — mirrors are used as base URLs for
|
||||
# constructing search requests; params/fragments on the configured URL
|
||||
# produce malformed URLs when paths are appended (issue #999).
|
||||
parsed = urlparse(normalized)
|
||||
if parsed.query or parsed.fragment:
|
||||
normalized = parsed._replace(query="", fragment="").geturl()
|
||||
|
||||
if strip_trailing_slash:
|
||||
normalized = normalized.rstrip("/")
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import math
|
||||
import shutil
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
@@ -55,6 +56,19 @@ SECONDS_PER_HOUR = 3600
|
||||
# How long to wait for completed files to appear (seconds)
|
||||
COMPLETED_PATH_RETRY_INTERVAL = 5
|
||||
COMPLETED_PATH_MAX_ATTEMPTS = 12 # 12 attempts * 5s = 60s grace period
|
||||
COMPLETED_PATH_TIMEOUT_SETTING = "DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT"
|
||||
COMPLETED_PATH_TIMEOUT_MAX_SECONDS = 3600
|
||||
_RETRYABLE_COMPLETED_PATH_ERRNOS = frozenset(
|
||||
code
|
||||
for code in (
|
||||
errno.ENOENT,
|
||||
getattr(errno, "ESTALE", None),
|
||||
getattr(errno, "EAGAIN", None),
|
||||
getattr(errno, "EBUSY", None),
|
||||
getattr(errno, "ETIMEDOUT", None),
|
||||
)
|
||||
if code is not None
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -69,6 +83,39 @@ class DownloadRequest:
|
||||
ratio_limit: float | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CompletedPathResolution:
|
||||
path: Path | None
|
||||
error: str | None
|
||||
retryable: bool
|
||||
|
||||
|
||||
def _coerce_completed_path_timeout_seconds(value: object, default: float) -> float:
|
||||
if isinstance(value, bool) or value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
parsed = float(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
parsed = float(value.strip())
|
||||
except ValueError:
|
||||
return default
|
||||
else:
|
||||
return default
|
||||
|
||||
if not math.isfinite(parsed) or parsed < 0:
|
||||
return default
|
||||
return min(parsed, float(COMPLETED_PATH_TIMEOUT_MAX_SECONDS))
|
||||
|
||||
|
||||
def _is_retryable_completed_path_probe(error: OSError | None) -> bool:
|
||||
return error is not None and error.errno in _RETRYABLE_COMPLETED_PATH_ERRNOS
|
||||
|
||||
|
||||
def _path_needs_mapping(path: str) -> bool:
|
||||
return (len(path) >= WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":") or "\\" in path
|
||||
|
||||
|
||||
def _diagnose_path_issue(path: str) -> str:
|
||||
"""Analyze a path and return diagnostic hints for common issues.
|
||||
|
||||
@@ -167,6 +214,12 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
"""Maximum attempts when waiting for completed files."""
|
||||
return COMPLETED_PATH_MAX_ATTEMPTS
|
||||
|
||||
def _completed_path_timeout_seconds(self) -> float:
|
||||
"""Total time to wait for completed files to appear on disk."""
|
||||
fallback = self._completed_path_retry_interval() * self._completed_path_max_attempts()
|
||||
configured = config.get(COMPLETED_PATH_TIMEOUT_SETTING, fallback)
|
||||
return _coerce_completed_path_timeout_seconds(configured, fallback)
|
||||
|
||||
def _get_category_for_task(self, client: DownloadClient, task: DownloadTask) -> str | None:
|
||||
"""Get audiobook category if configured and applicable, else None for default."""
|
||||
if not is_audiobook(task.content_type):
|
||||
@@ -387,6 +440,21 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
log_details: bool,
|
||||
) -> tuple[Path | None, str | None]:
|
||||
"""Resolve and validate the completed download path once."""
|
||||
result = self._resolve_download_path_once_detailed(
|
||||
client,
|
||||
download_id,
|
||||
log_details=log_details,
|
||||
)
|
||||
return result.path, result.error
|
||||
|
||||
def _resolve_download_path_once_detailed(
|
||||
self,
|
||||
client: DownloadClient,
|
||||
download_id: str,
|
||||
*,
|
||||
log_details: bool,
|
||||
) -> _CompletedPathResolution:
|
||||
"""Resolve and validate a completed path, including retryability."""
|
||||
try:
|
||||
raw_path = client.get_download_path(download_id)
|
||||
except Exception as e:
|
||||
@@ -402,7 +470,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.debug(
|
||||
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
|
||||
)
|
||||
return None, message
|
||||
return _CompletedPathResolution(None, message, retryable=False)
|
||||
|
||||
if not raw_path:
|
||||
message = (
|
||||
@@ -417,7 +485,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.debug(
|
||||
"Download client returned empty path for %s %s", client.name, download_id
|
||||
)
|
||||
return None, message
|
||||
return _CompletedPathResolution(None, message, retryable=False)
|
||||
|
||||
from shelfmark.core.path_mappings import (
|
||||
get_client_host_identifier,
|
||||
@@ -457,7 +525,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.error(failure_log, *failure_args)
|
||||
else:
|
||||
logger.debug(failure_log, *failure_args)
|
||||
return None, message
|
||||
return _CompletedPathResolution(None, message, retryable=False)
|
||||
|
||||
remapped_exists, remapped_error = _probe_completed_path(remapped)
|
||||
|
||||
@@ -480,7 +548,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
source_path_obj,
|
||||
remapped,
|
||||
)
|
||||
return remapped, None
|
||||
return _CompletedPathResolution(remapped, None, retryable=False)
|
||||
|
||||
message = (
|
||||
f"Remapped path '{remapped}' does not exist. "
|
||||
@@ -499,7 +567,11 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.error(failure_log, *failure_args)
|
||||
else:
|
||||
logger.debug(failure_log, *failure_args)
|
||||
return None, message
|
||||
return _CompletedPathResolution(
|
||||
None,
|
||||
message,
|
||||
retryable=_is_retryable_completed_path_probe(remapped_error),
|
||||
)
|
||||
|
||||
source_exists, source_error = _probe_completed_path(source_path_obj)
|
||||
|
||||
@@ -522,7 +594,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
download_id,
|
||||
source_path_obj,
|
||||
)
|
||||
return source_path_obj, None
|
||||
return _CompletedPathResolution(source_path_obj, None, retryable=False)
|
||||
|
||||
hint = _diagnose_path_issue(raw_path)
|
||||
if mappings:
|
||||
@@ -555,7 +627,12 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
logger.error(failure_log, *failure_args)
|
||||
else:
|
||||
logger.debug(failure_log, *failure_args)
|
||||
return None, message
|
||||
return _CompletedPathResolution(
|
||||
None,
|
||||
message,
|
||||
retryable=not _path_needs_mapping(raw_path)
|
||||
and _is_retryable_completed_path_probe(source_error),
|
||||
)
|
||||
|
||||
def _wait_for_completed_path(
|
||||
self,
|
||||
@@ -567,23 +644,37 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
) -> tuple[Path | None, str | None]:
|
||||
"""Wait briefly for completed files to appear on disk."""
|
||||
last_error: str | None = None
|
||||
max_attempts = self._completed_path_max_attempts()
|
||||
retry_interval = self._completed_path_retry_interval()
|
||||
timeout_seconds = self._completed_path_timeout_seconds()
|
||||
if retry_interval <= 0 or timeout_seconds <= 0:
|
||||
max_attempts = 1
|
||||
else:
|
||||
max_attempts = int(math.ceil(timeout_seconds / retry_interval)) + 1
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
if cancel_flag and cancel_flag.is_set():
|
||||
return None, last_error
|
||||
|
||||
log_details = attempt == max_attempts
|
||||
resolved_path, error = self._resolve_download_path_once(
|
||||
result = self._resolve_download_path_once_detailed(
|
||||
client,
|
||||
download_id,
|
||||
log_details=log_details,
|
||||
)
|
||||
if resolved_path:
|
||||
return resolved_path, None
|
||||
if result.path:
|
||||
return result.path, None
|
||||
|
||||
last_error = error
|
||||
last_error = result.error
|
||||
|
||||
if not result.retryable:
|
||||
if not log_details:
|
||||
logger.error(
|
||||
"Completed path resolution is not retryable for %s (%s): %s",
|
||||
client.name,
|
||||
download_id,
|
||||
last_error,
|
||||
)
|
||||
return None, last_error
|
||||
|
||||
if attempt < max_attempts:
|
||||
status_callback("locating", "Waiting for completed files...")
|
||||
|
||||
@@ -258,7 +258,7 @@ class QBittorrentClient(DownloadClient):
|
||||
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
|
||||
|
||||
def _get_torrents_info(
|
||||
self, torrent_hash: str | None = None
|
||||
self, torrent_hash: str | None = None, category: str | None = None
|
||||
) -> tuple[list[SimpleNamespace], str | None]:
|
||||
"""Get torrent info using GET.
|
||||
|
||||
@@ -267,6 +267,7 @@ class QBittorrentClient(DownloadClient):
|
||||
- Keep "API/auth/connect" errors distinct from "torrent missing".
|
||||
- If a hash-specific query returns empty, fall back to listing by category
|
||||
and matching locally.
|
||||
- Without a hash, `category` narrows the listing to that category.
|
||||
|
||||
Returns:
|
||||
(torrents, error_message)
|
||||
@@ -301,6 +302,8 @@ class QBittorrentClient(DownloadClient):
|
||||
primary_params: dict[str, str] = {}
|
||||
if torrent_hash:
|
||||
primary_params["hashes"] = torrent_hash
|
||||
elif category:
|
||||
primary_params["category"] = category
|
||||
|
||||
response = do_request(primary_params)
|
||||
torrents, error = parse_response(response, request_params=primary_params)
|
||||
@@ -358,6 +361,44 @@ class QBittorrentClient(DownloadClient):
|
||||
else:
|
||||
return torrents, None
|
||||
|
||||
def _list_category_hashes(self, category: str | None) -> set[str] | None:
|
||||
"""Snapshot the hashes qBittorrent currently reports for a category."""
|
||||
torrents, error = self._get_torrents_info(category=category)
|
||||
if error:
|
||||
logger.debug("Could not snapshot qBittorrent torrents: %s", error)
|
||||
return None
|
||||
return {str(torrent.hash).lower() for torrent in torrents if getattr(torrent, "hash", None)}
|
||||
|
||||
def _discover_added_torrent_hash(
|
||||
self,
|
||||
name: str,
|
||||
category: str | None,
|
||||
known_hashes: set[str] | None,
|
||||
) -> str | None:
|
||||
"""Recover the hash of a torrent that was added without a known info_hash.
|
||||
|
||||
A `known_hashes` of None means the pre-add snapshot failed, so only a
|
||||
torrent matching the requested rename can identify the new arrival.
|
||||
"""
|
||||
for _ in range(20):
|
||||
torrents, error = self._get_torrents_info(category=category)
|
||||
if error:
|
||||
logger.debug("qBittorrent hash discovery: %s", error)
|
||||
else:
|
||||
new_torrents = [
|
||||
torrent
|
||||
for torrent in torrents
|
||||
if getattr(torrent, "hash", None)
|
||||
and (known_hashes is None or str(torrent.hash).lower() not in known_hashes)
|
||||
]
|
||||
for torrent in new_torrents:
|
||||
if getattr(torrent, "name", None) == name:
|
||||
return str(torrent.hash).lower()
|
||||
if known_hashes is not None and len(new_torrents) == 1:
|
||||
return str(new_torrents[0].hash).lower()
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if qBittorrent is configured and selected as the torrent client."""
|
||||
@@ -425,6 +466,10 @@ class QBittorrentClient(DownloadClient):
|
||||
expected_hash = torrent_info.info_hash
|
||||
torrent_data = torrent_info.torrent_data
|
||||
|
||||
known_hashes: set[str] | None = None
|
||||
if not expected_hash:
|
||||
known_hashes = self._list_category_hashes(category)
|
||||
|
||||
# Per-torrent seeding limits from indexer
|
||||
seeding_time_limit_value = kwargs.get("seeding_time_limit")
|
||||
seeding_time_limit = coerce_optional_int(seeding_time_limit_value)
|
||||
@@ -459,12 +504,17 @@ class QBittorrentClient(DownloadClient):
|
||||
result_text = _normalize_add_result(result)
|
||||
logger.debug("qBittorrent add result: %s", result_text)
|
||||
|
||||
if not expected_hash:
|
||||
_raise_runtime_error("Could not determine torrent hash from URL")
|
||||
|
||||
if _is_explicit_add_failure(result):
|
||||
_raise_runtime_error(f"Failed to add torrent: {result_text}")
|
||||
|
||||
if not expected_hash:
|
||||
# qBittorrent fetches .torrent URLs itself, so the add can succeed
|
||||
# even when no hash could be extracted up front. Recover it by
|
||||
# watching for the new torrent to appear.
|
||||
expected_hash = self._discover_added_torrent_hash(name, category, known_hashes)
|
||||
if not expected_hash:
|
||||
_raise_runtime_error("Could not determine torrent hash from URL")
|
||||
|
||||
# Some qBittorrent-compatible clients return HTTP 200 with an empty body
|
||||
# instead of qBittorrent's literal "Ok." response. Prefer verifying that
|
||||
# the torrent becomes visible over trusting the response body alone.
|
||||
|
||||
@@ -4,6 +4,7 @@ Uses xmlrpc to communicate with rTorrent's RPC interface.
|
||||
"""
|
||||
|
||||
import ssl
|
||||
import time
|
||||
import xmlrpc.client as stdlib_xmlrpc_client
|
||||
from typing import Any, NoReturn, Protocol, cast
|
||||
from urllib.parse import urlparse
|
||||
@@ -115,6 +116,7 @@ class RTorrentClient(DownloadClient):
|
||||
self._rpc = _create_rtorrent_server_proxy(self._base_url)
|
||||
self._download_dir = config_text(config.get("RTORRENT_DOWNLOAD_DIR", ""))
|
||||
self._label = config_text(config.get("RTORRENT_LABEL", ""))
|
||||
self._audiobook_label = config_text(config.get("RTORRENT_AUDIOBOOK_LABEL", ""))
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
@@ -159,9 +161,17 @@ class RTorrentClient(DownloadClient):
|
||||
try:
|
||||
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
|
||||
known_hashes: set[str] | None = None
|
||||
if not (torrent_info.info_hash or expected_hash):
|
||||
known_hashes = self._list_torrent_hashes()
|
||||
|
||||
commands = []
|
||||
|
||||
label = category or self._label
|
||||
is_audiobook = kwargs.get("content_type") == "audiobook"
|
||||
default_label = (
|
||||
self._audiobook_label if is_audiobook and self._audiobook_label else self._label
|
||||
)
|
||||
label = category or default_label
|
||||
if label:
|
||||
logger.debug("Setting rTorrent label: %s", label)
|
||||
commands.append(f"d.custom1.set={label}")
|
||||
@@ -190,6 +200,11 @@ class RTorrentClient(DownloadClient):
|
||||
self._rpc.load.start("", add_url, ";".join(commands))
|
||||
|
||||
torrent_hash = torrent_info.info_hash or expected_hash
|
||||
if not torrent_hash:
|
||||
# rTorrent fetches .torrent URLs itself, so the add can succeed
|
||||
# even when no hash could be extracted up front. Recover it by
|
||||
# watching for the new download to appear.
|
||||
torrent_hash = self._discover_added_torrent_hash(name, label, known_hashes)
|
||||
if not torrent_hash:
|
||||
_raise_runtime_error("Could not determine torrent hash from URL")
|
||||
|
||||
@@ -382,6 +397,56 @@ class RTorrentClient(DownloadClient):
|
||||
except _RTORRENT_CLIENT_ERRORS:
|
||||
return "/downloads"
|
||||
|
||||
def _list_torrent_hashes(self) -> set[str] | None:
|
||||
"""Snapshot the hashes rTorrent currently reports."""
|
||||
try:
|
||||
all_torrents = self._rpc.d.multicall2("", "", "d.hash=")
|
||||
except _RTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("Could not snapshot rTorrent downloads: %s", e)
|
||||
return None
|
||||
return {str(row[0]).lower() for row in all_torrents if row and row[0]}
|
||||
|
||||
def _discover_added_torrent_hash(
|
||||
self,
|
||||
name: str,
|
||||
label: str,
|
||||
known_hashes: set[str] | None,
|
||||
) -> str | None:
|
||||
"""Recover the hash of a torrent that was added without a known info_hash.
|
||||
|
||||
rTorrent fetches .torrent URLs itself, so the add can succeed even when
|
||||
no hash could be extracted up front. A `known_hashes` of None means the
|
||||
pre-add snapshot failed, so only an exact name match can identify the
|
||||
new arrival.
|
||||
"""
|
||||
for _ in range(20):
|
||||
try:
|
||||
all_torrents = self._rpc.d.multicall2("", "", "d.hash=", "d.name=", "d.custom1=")
|
||||
except _RTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("rTorrent hash discovery: %s", e)
|
||||
else:
|
||||
new_torrents = [
|
||||
row
|
||||
for row in all_torrents
|
||||
if row
|
||||
and row[0]
|
||||
and (known_hashes is None or str(row[0]).lower() not in known_hashes)
|
||||
]
|
||||
# The label set at add time distinguishes concurrent arrivals,
|
||||
# but rTorrent may not have applied it yet, so it only ever
|
||||
# narrows a non-empty candidate list.
|
||||
if label:
|
||||
labeled = [row for row in new_torrents if len(row) > 2 and row[2] == label]
|
||||
if labeled:
|
||||
new_torrents = labeled
|
||||
for row in new_torrents:
|
||||
if len(row) > 1 and row[1] == name:
|
||||
return str(row[0]).lower()
|
||||
if known_hashes is not None and len(new_torrents) == 1:
|
||||
return str(new_torrents[0][0]).lower()
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
|
||||
def _get_torrent_path(self, download_id: str) -> str | None:
|
||||
"""Get the file path of a torrent by hash.
|
||||
|
||||
|
||||
@@ -748,11 +748,18 @@ def prowlarr_clients_settings() -> list[SettingsField]:
|
||||
TextField(
|
||||
key="RTORRENT_LABEL",
|
||||
label="Book Label",
|
||||
description="Label to assign to book downloads in rTorrent",
|
||||
description="Label to assign to ebook downloads in rTorrent",
|
||||
placeholder="cwabd",
|
||||
default="cwabd",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
TextField(
|
||||
key="RTORRENT_AUDIOBOOK_LABEL",
|
||||
label="Audiobook Label",
|
||||
description="Label to assign to audiobook downloads in rTorrent (falls back to Book Label if not set)",
|
||||
placeholder="audiobooks",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
TextField(
|
||||
key="RTORRENT_DOWNLOAD_DIR",
|
||||
label="Download Directory",
|
||||
|
||||
@@ -19,6 +19,7 @@ from shelfmark.download.network import get_ssl_verify
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
_MAGNET_RESPONSE_MAX_BYTES = 2000
|
||||
_TORRENT_FETCH_MAX_REDIRECTS = 5
|
||||
_BASE32_BTMH_TAG_BYTES = 34
|
||||
_BTIH_INFO_BYTE_HEX = 0x20
|
||||
_BTIH_PREFIX_BYTE = 0x12
|
||||
@@ -95,10 +96,16 @@ def extract_torrent_info(
|
||||
# Not a magnet - try to fetch and parse the .torrent file
|
||||
if not fetch_torrent:
|
||||
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
|
||||
if not _is_trusted_torrent_fetch_url(url):
|
||||
logger.debug("Skipping torrent prefetch for untrusted URL: %s...", url[:80])
|
||||
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
|
||||
|
||||
# A release source can legitimately hand us a download URL on a different
|
||||
# origin than the configured Prowlarr/Newznab endpoint (e.g. a direct
|
||||
# tracker link, or Prowlarr reached through a separate proxy), and a trusted
|
||||
# Prowlarr download URL commonly redirects to the indexer's own download
|
||||
# link. We still need to fetch the .torrent to recover the info_hash when
|
||||
# the source did not provide one, so the prefetch runs regardless of origin
|
||||
# and follows cross-origin redirects. The Prowlarr API key, however, is
|
||||
# re-evaluated per hop and only ever sent to a trusted origin so it can
|
||||
# never leak to an arbitrary indexer/tracker host.
|
||||
headers: dict[str, str] = {"Accept": "application/x-bittorrent"}
|
||||
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
|
||||
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
|
||||
@@ -114,19 +121,28 @@ def extract_torrent_info(
|
||||
try:
|
||||
logger.debug("Fetching torrent file from: %s...", url[:80])
|
||||
|
||||
# Use allow_redirects=False to handle magnet link redirects manually
|
||||
# Some indexers redirect download URLs to magnet links
|
||||
resp = requests.get(
|
||||
url,
|
||||
timeout=30,
|
||||
allow_redirects=False,
|
||||
headers=headers,
|
||||
verify=get_ssl_verify(url),
|
||||
)
|
||||
# Redirects are followed manually: some indexers redirect download URLs
|
||||
# to magnet links, and each hop must decide anew whether it may see the
|
||||
# API key.
|
||||
current_url = url
|
||||
redirects_remaining = _TORRENT_FETCH_MAX_REDIRECTS
|
||||
while True:
|
||||
request_headers = dict(headers)
|
||||
if not _is_trusted_torrent_fetch_url(current_url):
|
||||
request_headers.pop("X-Api-Key", None)
|
||||
|
||||
# Check if this is a redirect to a magnet link
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
redirect_url = resolve_url(url, resp.headers.get("Location", ""))
|
||||
resp = requests.get(
|
||||
current_url,
|
||||
timeout=30,
|
||||
allow_redirects=False,
|
||||
headers=request_headers,
|
||||
verify=get_ssl_verify(current_url),
|
||||
)
|
||||
|
||||
if resp.status_code not in (301, 302, 303, 307, 308):
|
||||
break
|
||||
|
||||
redirect_url = resolve_url(current_url, resp.headers.get("Location", ""))
|
||||
if redirect_url.startswith("magnet:"):
|
||||
logger.debug("Download URL redirected to magnet link")
|
||||
info_hash = extract_hash_from_magnet(redirect_url)
|
||||
@@ -138,20 +154,12 @@ def extract_torrent_info(
|
||||
is_magnet=True,
|
||||
magnet_url=redirect_url,
|
||||
)
|
||||
if not _is_trusted_torrent_fetch_url(redirect_url):
|
||||
logger.debug(
|
||||
"Skipping torrent prefetch redirect to untrusted URL: %s...",
|
||||
redirect_url[:80],
|
||||
)
|
||||
if redirects_remaining <= 0:
|
||||
logger.debug("Too many redirects fetching torrent file: %s...", url[:80])
|
||||
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
|
||||
# Not a magnet redirect, follow it manually
|
||||
redirects_remaining -= 1
|
||||
logger.debug("Following redirect to: %s...", redirect_url[:80])
|
||||
resp = requests.get(
|
||||
redirect_url,
|
||||
timeout=30,
|
||||
headers=headers,
|
||||
verify=get_ssl_verify(redirect_url),
|
||||
)
|
||||
current_url = redirect_url
|
||||
|
||||
resp.raise_for_status()
|
||||
torrent_data = resp.content
|
||||
|
||||
@@ -316,8 +316,12 @@ class TransmissionClient(DownloadClient):
|
||||
|
||||
state, message = status_map.get(status_value, ("downloading", "Downloading"))
|
||||
progress = torrent.percent_done * 100
|
||||
# Only mark complete when seeding - seed pending means files still being moved
|
||||
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value == "seeding"
|
||||
# Only mark complete when seeding or stopped (e.g. if seed limit/ratio is 0)
|
||||
# and progress is complete. seed pending means files still being moved
|
||||
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value in (
|
||||
"seeding",
|
||||
"stopped",
|
||||
)
|
||||
|
||||
if complete:
|
||||
message = "Complete"
|
||||
|
||||
@@ -852,6 +852,99 @@ def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver:
|
||||
return custom_resolver
|
||||
|
||||
|
||||
# --- ISP / network DNS interference detection ---------------------------------
|
||||
# Compare what the (tamperable) system resolver returns for a host against a
|
||||
# tamper-resistant DoH lookup. Divergent answers are a strong signal the network is
|
||||
# hijacking or NXDOMAIN-blocking the domain (a common reason AA downloads "work" but
|
||||
# land on an ISP block page). Used to surface an actionable hint to the user.
|
||||
_dns_interference_warned: set[str] = set()
|
||||
_dns_interference_active = False
|
||||
|
||||
|
||||
def _build_detection_doh_resolver() -> DoHResolver | None:
|
||||
"""Build a throwaway DoH resolver for interference checks (no socket patching).
|
||||
|
||||
Honours the DoH provider the user selected (``DNS_PROVIDERS[_current_dns_index]``),
|
||||
falling back to the first configured provider when none is active. The endpoint is
|
||||
pinned to the provider's own nameserver IP so resolving the DoH host can't be
|
||||
redirected by the very DNS layer the check is meant to detect.
|
||||
"""
|
||||
if 0 <= _current_dns_index < len(DNS_PROVIDERS):
|
||||
_name, servers, doh_url = DNS_PROVIDERS[_current_dns_index]
|
||||
elif DNS_PROVIDERS:
|
||||
_name, servers, doh_url = DNS_PROVIDERS[0]
|
||||
else:
|
||||
return None
|
||||
server_hostname = urllib.parse.urlparse(doh_url).hostname or ""
|
||||
if not server_hostname or not servers:
|
||||
return None
|
||||
return DoHResolver(doh_url, server_hostname, servers[0])
|
||||
|
||||
|
||||
def detect_dns_interference(hostname: str) -> dict[str, list[str]] | None:
|
||||
"""Detect network DNS interference by comparing system DNS against DoH.
|
||||
|
||||
Returns ``{"system_ips": [...], "doh_ips": [...]}`` when the two resolvers disagree
|
||||
(no overlapping IPs), otherwise None. No-op for IP literals / local hostnames and
|
||||
when DoH resolution is unavailable, so it never produces a false positive.
|
||||
"""
|
||||
host = (hostname or "").strip().lower()
|
||||
if not host or _is_ip_address(host) or _is_local_address(host):
|
||||
return None
|
||||
resolver = _build_detection_doh_resolver()
|
||||
if resolver is None:
|
||||
return None
|
||||
try:
|
||||
system_ips = {str(info[4][0]) for info in original_getaddrinfo(host, 443, socket.AF_INET)}
|
||||
except OSError:
|
||||
return None
|
||||
if not system_ips:
|
||||
return None
|
||||
doh_ips = {ip for ip in resolver.resolve(host, "A") if ip}
|
||||
if not doh_ips or (system_ips & doh_ips):
|
||||
return None
|
||||
return {"system_ips": sorted(system_ips), "doh_ips": sorted(doh_ips)}
|
||||
|
||||
|
||||
def note_possible_dns_interference(hostname: str) -> bool:
|
||||
"""Check ``hostname`` for DNS interference, logging an actionable warning once.
|
||||
|
||||
Returns True when interference has been detected this session. The check runs at
|
||||
most once per host to avoid repeated DoH lookups and log spam.
|
||||
"""
|
||||
global _dns_interference_active
|
||||
host = (hostname or "").strip().lower()
|
||||
if not host or host in _dns_interference_warned:
|
||||
return _dns_interference_active
|
||||
_dns_interference_warned.add(host)
|
||||
|
||||
result = detect_dns_interference(host)
|
||||
if not result:
|
||||
return _dns_interference_active
|
||||
|
||||
_dns_interference_active = True
|
||||
routing_via_doh = _current_dns_index >= 0 and bool(DOH_SERVER)
|
||||
remedy = (
|
||||
"Shelfmark is routing this domain through DNS-over-HTTPS to work around it."
|
||||
if routing_via_doh
|
||||
else "Enable DNS-over-HTTPS (USE_DOH=true) or set a custom DNS provider to bypass it."
|
||||
)
|
||||
logger.warning(
|
||||
"Possible ISP/network DNS interference for %s: system DNS resolves to %s but DoH "
|
||||
"resolves to %s. The network appears to be blocking or redirecting this domain. %s",
|
||||
host,
|
||||
result["system_ips"],
|
||||
result["doh_ips"],
|
||||
remedy,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def dns_interference_detected() -> bool:
|
||||
"""Whether network DNS interference has been detected this session."""
|
||||
return _dns_interference_active
|
||||
|
||||
|
||||
def init_doh_resolver(doh_server: str = "") -> DoHResolver | None:
|
||||
"""Initialize DNS over HTTPS resolver."""
|
||||
server = doh_server or DOH_SERVER
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -40,9 +41,11 @@ def validate_destination(
|
||||
status_callback("error", f"Destination is not a directory: {destination}")
|
||||
return False
|
||||
|
||||
created_by_us = False
|
||||
if not destination_exists:
|
||||
try:
|
||||
run_blocking_io(destination.mkdir, parents=True, exist_ok=True)
|
||||
created_by_us = True
|
||||
except (OSError, PermissionError) as exc:
|
||||
log_path_permission_context("destination_create", destination)
|
||||
logger.warning("Cannot create destination: %s (%s)", destination, exc)
|
||||
@@ -63,6 +66,9 @@ def validate_destination(
|
||||
log_path_permission_context("destination_write_probe", destination)
|
||||
logger.warning("Destination not writable: %s (%s)", destination, exc)
|
||||
status_callback("error", f"Destination not writable: {destination} ({exc})")
|
||||
if created_by_us:
|
||||
with contextlib.suppress(OSError):
|
||||
run_blocking_io(destination.rmdir)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@@ -1163,6 +1163,9 @@ def api_config() -> Response | tuple[Response, int]:
|
||||
"show_combined_selector": app_config.get(
|
||||
"SHOW_COMBINED_SELECTOR", True, user_id=db_user_id
|
||||
),
|
||||
"force_combined_search": app_config.get(
|
||||
"FORCE_COMBINED_SEARCH", False, user_id=db_user_id
|
||||
),
|
||||
"books_output_mode": app_config.get("BOOKS_OUTPUT_MODE", "folder"),
|
||||
"auto_open_downloads_sidebar": app_config.get("AUTO_OPEN_DOWNLOADS_SIDEBAR", True),
|
||||
"hardcover_auto_remove_on_download": app_config.get(
|
||||
|
||||
@@ -2628,7 +2628,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
raise RuntimeError(msg) from e
|
||||
return None
|
||||
except requests.HTTPError as e:
|
||||
if e.response.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
if e.response is not None and e.response.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
logger.exception("Hardcover API key is invalid")
|
||||
if raise_on_error:
|
||||
msg = "Hardcover API key is invalid"
|
||||
|
||||
@@ -214,7 +214,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
logger.warning("Open Library search timed out")
|
||||
return []
|
||||
except requests.HTTPError as e:
|
||||
if e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
|
||||
if e.response is not None and e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
|
||||
logger.warning("Open Library service unavailable (503)")
|
||||
else:
|
||||
logger.exception("Open Library HTTP error")
|
||||
@@ -253,7 +253,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
logger.warning("Open Library get_book timed out")
|
||||
return None
|
||||
except requests.HTTPError as e:
|
||||
if e.response.status_code == HTTPStatus.NOT_FOUND:
|
||||
if e.response is not None and e.response.status_code == HTTPStatus.NOT_FOUND:
|
||||
logger.debug("Open Library work not found: %s", book_id)
|
||||
else:
|
||||
logger.exception("Open Library HTTP error")
|
||||
@@ -314,7 +314,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
return self._parse_edition(edition, clean_isbn)
|
||||
|
||||
except requests.HTTPError as e:
|
||||
if e.response.status_code == HTTPStatus.NOT_FOUND:
|
||||
if e.response is not None and e.response.status_code == HTTPStatus.NOT_FOUND:
|
||||
logger.debug("Open Library ISBN not found: %s", isbn)
|
||||
else:
|
||||
logger.exception("Open Library ISBN search HTTP error")
|
||||
|
||||
@@ -417,6 +417,18 @@ def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") ->
|
||||
# Clean up info hash (remove whitespace, ensure uppercase)
|
||||
info_hash = re.sub(r"\s+", "", info_hash).upper()
|
||||
|
||||
# Validate: SHA1 = 40 hex chars, SHA256 = 64 hex chars
|
||||
if not re.match(r"^[0-9A-F]{40}$|^[0-9A-F]{64}$", info_hash):
|
||||
logger.warning("Info Hash invalid (got %r), trying magnet fallback.", info_hash)
|
||||
# Fallback: search entire page for a complete magnet link (e.g. posted in comments)
|
||||
magnet_match = re.search(r"magnet:\?xt=urn:btih:([0-9a-fA-F]{40,64})", detail_html)
|
||||
if magnet_match:
|
||||
info_hash = magnet_match.group(1).upper()
|
||||
logger.info("Found hash via magnet fallback: %s", info_hash)
|
||||
else:
|
||||
logger.warning("No valid magnet link found on page, giving up.")
|
||||
return None
|
||||
|
||||
# 2. Extract Trackers
|
||||
# Find all <td> containing udp:// or http://
|
||||
trackers = []
|
||||
|
||||
@@ -238,8 +238,8 @@ class AudiobookBaySource(ReleaseSource):
|
||||
exact_phrase=exact_phrase,
|
||||
)
|
||||
|
||||
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
|
||||
if exact_phrase and not results and not plan.manual_query:
|
||||
# Fallback to broad matching if exact phrase returns nothing (manual or auto query).
|
||||
if exact_phrase and not results:
|
||||
logger.info(
|
||||
"No exact phrase results, retrying AudiobookBay search without quotes"
|
||||
)
|
||||
@@ -288,7 +288,7 @@ class AudiobookBaySource(ReleaseSource):
|
||||
size_str = result.get("size")
|
||||
size_bytes = parse_size(size_str) if size_str else None
|
||||
language_raw = result.get("language")
|
||||
language_code = _map_language(language_raw) if language_raw else None
|
||||
language_code = _map_language(language_raw) if language_raw else "en"
|
||||
bitrate = result.get("bitrate")
|
||||
bitrate_kbps = _parse_bitrate_to_kbps(bitrate)
|
||||
|
||||
|
||||
@@ -3,9 +3,12 @@
|
||||
import itertools
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
from dataclasses import replace
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar, NoReturn, TypedDict
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
@@ -197,6 +200,49 @@ _SOURCE_FAILURE_THRESHOLD = 4
|
||||
_MIN_VALID_FILE_SIZE = 10 * 1024
|
||||
_AA_COUNTDOWN_MAX_SECONDS = 300
|
||||
|
||||
# --- Distant-path language detection ---
|
||||
|
||||
_DISTANT_PATH_EXTENSIONS = (
|
||||
"epub",
|
||||
"mobi",
|
||||
"azw3",
|
||||
"fb2",
|
||||
"djvu",
|
||||
"cbz",
|
||||
"cbr",
|
||||
"pdf",
|
||||
"zip",
|
||||
"rar",
|
||||
"m4b",
|
||||
"mp3",
|
||||
)
|
||||
_DISTANT_PATH_EXTENSION_PATTERN = "|".join(re.escape(e) for e in _DISTANT_PATH_EXTENSIONS)
|
||||
_DISTANT_PATH_PATTERN = re.compile(
|
||||
rf"(?:[A-Za-z0-9._-]+/)?[A-Za-z]:(?:\\|/)[^\n\r<>\"]+?\.(?:{_DISTANT_PATH_EXTENSION_PATTERN})\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DISTANT_PATH_FALLBACK_PATTERN = re.compile(
|
||||
r"(?:[A-Za-z0-9._-]+/)?[A-Za-z]:(?:\\|/)[^\n\r<>\"]+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_BRACKETED_LANGUAGE_CODE_PATTERN = re.compile(
|
||||
r"\[(?:bd[\s._-]*)?([A-Za-z]{2,3})\]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_KEYED_LANGUAGE_CODE_PATTERN = re.compile(
|
||||
r"\b(?:bd|lang(?:uage)?)\s*[:._-]?\s*([A-Za-z]{2,3})\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_LANGUAGE_CODE_TOKEN_PATTERN = re.compile(
|
||||
r"(?:^|[\s_./\\\-\[(])([A-Za-z]{2,3})(?=$|[\s_./\\\-)\]])"
|
||||
)
|
||||
_LANGUAGE_NAME_TOKEN_PATTERN = re.compile(r"[a-z]{4,}(?:-[a-z0-9]+)?")
|
||||
_LANGUAGE_ALIAS_TO_CODE: dict[str, str] | None = None
|
||||
_LANGUAGE_ALIAS_LOCK = threading.Lock()
|
||||
_LANGUAGE_PLACEHOLDERS = frozenset({"", "-", "--", "unknown", "unk", "n/a", "na"})
|
||||
# Short codes that appear in common words — require bracket/key context to accept
|
||||
_AMBIGUOUS_SHORT_LANGUAGE_CODES = frozenset({"de", "en", "it", "la", "no", "or", "is", "in"})
|
||||
|
||||
# Sources that require Cloudflare bypass
|
||||
_CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "welib"})
|
||||
|
||||
@@ -204,6 +250,189 @@ _CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "weli
|
||||
_AA_PAGE_SOURCES = frozenset({"aa-slow-nowait", "aa-slow-wait"})
|
||||
|
||||
|
||||
def _is_language_from_path_enabled() -> bool:
|
||||
return bool(config.get("DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH", False))
|
||||
|
||||
|
||||
def _normalize_language_token(value: str) -> str:
|
||||
normalized = value.strip().lower()
|
||||
for dash in ("‑", "–", "—", "−"):
|
||||
normalized = normalized.replace(dash, "-")
|
||||
return normalized
|
||||
|
||||
|
||||
def _fold_text(value: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKD", value)
|
||||
return "".join(c for c in normalized if not unicodedata.combining(c)).lower()
|
||||
|
||||
|
||||
def _language_alias_to_code() -> dict[str, str]:
|
||||
"""Build alias→code map from bundled language metadata (lazy, cached)."""
|
||||
global _LANGUAGE_ALIAS_TO_CODE
|
||||
cached = _LANGUAGE_ALIAS_TO_CODE
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
with _LANGUAGE_ALIAS_LOCK:
|
||||
cached = _LANGUAGE_ALIAS_TO_CODE
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
mapping: dict[str, str] = {}
|
||||
data_path = Path(__file__).resolve().parents[2] / "data" / "book-languages.json"
|
||||
|
||||
try:
|
||||
raw = json.loads(data_path.read_text(encoding="utf-8"))
|
||||
except OSError, ValueError, TypeError:
|
||||
_LANGUAGE_ALIAS_TO_CODE = {}
|
||||
return _LANGUAGE_ALIAS_TO_CODE
|
||||
|
||||
if not isinstance(raw, list):
|
||||
_LANGUAGE_ALIAS_TO_CODE = {}
|
||||
return _LANGUAGE_ALIAS_TO_CODE
|
||||
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
code = _normalize_language_token(str(item.get("code", "")))
|
||||
name = _normalize_language_token(str(item.get("language", "")))
|
||||
if not code:
|
||||
continue
|
||||
mapping.setdefault(code, code)
|
||||
mapping.setdefault(code.replace("-", "_"), code)
|
||||
mapping.setdefault(code.split("-")[0], code)
|
||||
mapping.setdefault(_fold_text(code), code)
|
||||
if name:
|
||||
mapping.setdefault(name, code)
|
||||
mapping.setdefault(_fold_text(name), code)
|
||||
|
||||
_LANGUAGE_ALIAS_TO_CODE = mapping
|
||||
return _LANGUAGE_ALIAS_TO_CODE
|
||||
|
||||
|
||||
def _extract_distant_path(row: Tag, *, enabled: bool) -> str | None:
|
||||
"""Extract the Windows-style file path from an AA search result row."""
|
||||
if not enabled:
|
||||
return None
|
||||
|
||||
def _normalize_candidate(text: str) -> str:
|
||||
normalized = re.sub(r"\s*([\\/])\s*", r"\1", text)
|
||||
normalized = re.sub(r":\s*([\\/])", r":\1", normalized)
|
||||
return re.sub(
|
||||
r"\s+\.(epub|mobi|azw3|fb2|djvu|cbz|cbr|pdf|zip|rar|m4b|mp3)\b",
|
||||
r".\1",
|
||||
normalized,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
candidates = [row.get_text(" ", strip=True)]
|
||||
for cell in row.find_all("td"):
|
||||
cell_text = cell.get_text(" ", strip=True)
|
||||
if cell_text:
|
||||
candidates.append(cell_text)
|
||||
|
||||
best: str | None = None
|
||||
for text in candidates:
|
||||
for match in _DISTANT_PATH_PATTERN.findall(_normalize_candidate(text)):
|
||||
candidate = match.strip().rstrip(".,;")
|
||||
if best is None or len(candidate) > len(best):
|
||||
best = candidate
|
||||
|
||||
if best is not None:
|
||||
return best
|
||||
|
||||
for text in candidates:
|
||||
for match in _DISTANT_PATH_FALLBACK_PATTERN.findall(_normalize_candidate(text)):
|
||||
candidate = match.strip().rstrip(".,;")
|
||||
if best is None or len(candidate) > len(best):
|
||||
best = candidate
|
||||
|
||||
return best
|
||||
|
||||
|
||||
def _detect_language_from_distant_path(path: str | None) -> str | None:
|
||||
"""Infer a language code from distant-path tags such as [BD FR] or [Fr]."""
|
||||
if not path:
|
||||
return None
|
||||
|
||||
aliases = _language_alias_to_code()
|
||||
if not aliases:
|
||||
return None
|
||||
|
||||
folded_path = _fold_text(path)
|
||||
strong_candidates: list[str] = []
|
||||
|
||||
for code in _BRACKETED_LANGUAGE_CODE_PATTERN.findall(path):
|
||||
normalized = _normalize_language_token(code)
|
||||
if normalized in aliases:
|
||||
strong_candidates.append(aliases[normalized])
|
||||
|
||||
for code in _KEYED_LANGUAGE_CODE_PATTERN.findall(path):
|
||||
normalized = _normalize_language_token(code)
|
||||
if normalized in aliases:
|
||||
strong_candidates.append(aliases[normalized])
|
||||
|
||||
non_ambiguous = [c for c in strong_candidates if c not in _AMBIGUOUS_SHORT_LANGUAGE_CODES]
|
||||
if non_ambiguous:
|
||||
return non_ambiguous[0]
|
||||
|
||||
for token in _LANGUAGE_NAME_TOKEN_PATTERN.findall(folded_path):
|
||||
normalized = _normalize_language_token(token)
|
||||
if normalized in aliases:
|
||||
candidate = aliases[normalized]
|
||||
if candidate not in _AMBIGUOUS_SHORT_LANGUAGE_CODES:
|
||||
return candidate
|
||||
|
||||
if strong_candidates:
|
||||
return strong_candidates[0]
|
||||
|
||||
for code in _LANGUAGE_CODE_TOKEN_PATTERN.findall(path):
|
||||
normalized = _normalize_language_token(code)
|
||||
if normalized in _AMBIGUOUS_SHORT_LANGUAGE_CODES:
|
||||
continue
|
||||
if normalized in aliases:
|
||||
return aliases[normalized]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _is_missing_or_placeholder_language(language: str | None) -> bool:
|
||||
if language is None:
|
||||
return True
|
||||
return _normalize_language_token(language) in _LANGUAGE_PLACEHOLDERS
|
||||
|
||||
|
||||
def _normalize_requested_languages(languages: list[str] | None) -> set[str]:
|
||||
if not languages:
|
||||
return set()
|
||||
aliases = _language_alias_to_code()
|
||||
normalized: set[str] = set()
|
||||
for value in languages:
|
||||
token = _normalize_language_token(str(value))
|
||||
if not token or token == "all": # noqa: S105 - "all" is a language sentinel
|
||||
continue
|
||||
normalized.add(aliases.get(token, token))
|
||||
return normalized
|
||||
|
||||
|
||||
def _book_matches_requested_languages(book_language: str | None, requested: set[str]) -> bool:
|
||||
"""Return True when a book's language matches the requested filter.
|
||||
|
||||
Books with unknown/missing language always pass — the server-side &lang= filter
|
||||
already narrowed the result set, so dropping unlabelled rows hides valid results.
|
||||
"""
|
||||
if not requested:
|
||||
return True
|
||||
if not book_language:
|
||||
return True
|
||||
aliases = _language_alias_to_code()
|
||||
normalized_book = aliases.get(
|
||||
_normalize_language_token(book_language),
|
||||
_normalize_language_token(book_language),
|
||||
)
|
||||
return normalized_book in requested
|
||||
|
||||
|
||||
def _is_configured_zlib_link(url: str) -> bool:
|
||||
"""Return True when a URL belongs to a configured Z-Library mirror."""
|
||||
from shelfmark.core.mirrors import get_zlib_cookie_domains
|
||||
@@ -360,9 +589,17 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
|
||||
filters_query = ""
|
||||
|
||||
for value in filters.lang or []:
|
||||
if value and value != "all":
|
||||
filters_query += f"&lang={quote(value)}"
|
||||
path_language_enabled = _is_language_from_path_enabled()
|
||||
requested_langs = _normalize_requested_languages(filters.lang)
|
||||
|
||||
# When path-language inference is on and a language is requested, skip the
|
||||
# server-side &lang= filter: lgli files often have no AA language metadata
|
||||
# and would be excluded before we can infer language from the distant path.
|
||||
# Local filtering below handles the narrowing instead.
|
||||
if not (path_language_enabled and requested_langs):
|
||||
for value in filters.lang or []:
|
||||
if value and value != "all":
|
||||
filters_query += f"&lang={quote(value)}"
|
||||
|
||||
if filters.sort and filters.sort != "relevance":
|
||||
filters_query += f"&sort={quote(filters.sort)}"
|
||||
@@ -397,14 +634,13 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
|
||||
if "No files found." in html:
|
||||
logger.info("No books found for query: %s", query)
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
tbody = soup.find("table")
|
||||
|
||||
if tbody is None:
|
||||
if "No files found." in html:
|
||||
logger.info("No books found for query: %s", query)
|
||||
return []
|
||||
logger.warning("No results table found for query: %s", query)
|
||||
msg = "No books found. Please try another query."
|
||||
raise RuntimeError(msg)
|
||||
@@ -418,6 +654,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
if book:
|
||||
books.append(book)
|
||||
|
||||
if path_language_enabled and requested_langs:
|
||||
books = [b for b in books if _book_matches_requested_languages(b.language, requested_langs)]
|
||||
|
||||
supported_formats = _get_supported_formats()
|
||||
|
||||
books.sort(
|
||||
@@ -471,10 +710,23 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
|
||||
if not record_id:
|
||||
return None
|
||||
|
||||
path_language_enabled = _is_language_from_path_enabled()
|
||||
distant_path = _extract_distant_path(row, enabled=path_language_enabled)
|
||||
|
||||
preview_img = cells[0].find("img")
|
||||
preview = _get_attr(preview_img, "src") if isinstance(preview_img, Tag) else None
|
||||
|
||||
title = _first_stripped_text(cells[1].find("span"))
|
||||
title_span = cells[1].find("span")
|
||||
if isinstance(title_span, Tag):
|
||||
# AA nests related-edition spans inside the main title span — take only direct text.
|
||||
direct = " ".join(
|
||||
str(c).strip()
|
||||
for c in title_span.children
|
||||
if isinstance(c, NavigableString) and str(c).strip()
|
||||
).strip()
|
||||
title = direct or _first_stripped_text(title_span)
|
||||
else:
|
||||
title = None
|
||||
author = _first_stripped_text(cells[2].find("span"))
|
||||
publisher = _first_stripped_text(cells[3].find("span"))
|
||||
year = _first_stripped_text(cells[4].find("span"))
|
||||
@@ -483,18 +735,19 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
|
||||
file_format = _first_stripped_text(cells[9].find("span"))
|
||||
size = _first_stripped_text(cells[10].find("span"))
|
||||
|
||||
if (
|
||||
title is None
|
||||
or author is None
|
||||
or publisher is None
|
||||
or year is None
|
||||
or language is None
|
||||
or content is None
|
||||
or file_format is None
|
||||
or size is None
|
||||
):
|
||||
# Only title and format are truly required — lgli rows often have sparse metadata
|
||||
if title is None or file_format is None:
|
||||
return None
|
||||
|
||||
# Skip entries where the title is a catalog format descriptor, not a real title
|
||||
# e.g. "Book/Online Audio", "Print book" — lgli metadata pollution
|
||||
if title and "/" in title and len(title) < 40 and not any(c.isdigit() for c in title):
|
||||
return None
|
||||
|
||||
if path_language_enabled and _is_missing_or_placeholder_language(language):
|
||||
detected = _detect_language_from_distant_path(distant_path)
|
||||
language = detected or "unknown"
|
||||
|
||||
return BrowseRecord(
|
||||
id=record_id,
|
||||
title=title,
|
||||
@@ -507,6 +760,7 @@ def _parse_search_result_row(row: Tag) -> BrowseRecord | None:
|
||||
content=content.lower() if content else None,
|
||||
format=file_format.lower() if file_format else None,
|
||||
size=size,
|
||||
download_path=distant_path,
|
||||
)
|
||||
except (AttributeError, IndexError, KeyError, TypeError) as e:
|
||||
logger.error_trace(f"Error parsing search result row: {e}")
|
||||
@@ -1229,6 +1483,9 @@ def _get_download_url(
|
||||
return downloader.get_absolute_url(link, url)
|
||||
|
||||
|
||||
_AA_COUNTDOWN_MAX_RETRIES = 3
|
||||
|
||||
|
||||
def _extract_slow_download_url(
|
||||
soup: BeautifulSoup,
|
||||
link: str,
|
||||
@@ -1237,6 +1494,7 @@ def _extract_slow_download_url(
|
||||
status_callback: Callable[[str, str | None], None] | None,
|
||||
selector: network.AAMirrorSelector,
|
||||
source_context: str | None = None,
|
||||
_countdown_attempts: int = 0,
|
||||
) -> str:
|
||||
"""Extract download URL from AA slow download pages."""
|
||||
html_str = str(soup)
|
||||
@@ -1301,6 +1559,14 @@ def _extract_slow_download_url(
|
||||
|
||||
countdown_seconds = _extract_countdown_seconds(soup, html_str)
|
||||
if countdown_seconds > 0:
|
||||
if _countdown_attempts >= _AA_COUNTDOWN_MAX_RETRIES:
|
||||
logger.warning(
|
||||
"Countdown retry limit (%s) reached for %s, giving up",
|
||||
_AA_COUNTDOWN_MAX_RETRIES,
|
||||
title,
|
||||
)
|
||||
return ""
|
||||
|
||||
max_countdown_seconds = 600
|
||||
sleep_time = min(countdown_seconds, max_countdown_seconds)
|
||||
if countdown_seconds > max_countdown_seconds:
|
||||
@@ -1309,7 +1575,13 @@ def _extract_slow_download_url(
|
||||
countdown_seconds,
|
||||
max_countdown_seconds,
|
||||
)
|
||||
logger.info("AA waitlist: %ss for %s", sleep_time, title)
|
||||
logger.info(
|
||||
"AA waitlist: %ss for %s (attempt %s/%s)",
|
||||
sleep_time,
|
||||
title,
|
||||
_countdown_attempts + 1,
|
||||
_AA_COUNTDOWN_MAX_RETRIES,
|
||||
)
|
||||
|
||||
# Live countdown with status updates
|
||||
for remaining in range(sleep_time, 0, -1):
|
||||
@@ -1330,12 +1602,31 @@ def _extract_slow_download_url(
|
||||
if status_callback and source_context:
|
||||
status_callback("resolving", f"{source_context} - Fetching")
|
||||
|
||||
return _get_download_url(
|
||||
link, title, cancel_flag, status_callback, selector, source_context
|
||||
html = downloader.html_get_page(
|
||||
link, selector=selector, cancel_flag=cancel_flag, status_callback=status_callback
|
||||
)
|
||||
if not html:
|
||||
return ""
|
||||
new_soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
return _extract_slow_download_url(
|
||||
new_soup,
|
||||
link,
|
||||
title,
|
||||
cancel_flag,
|
||||
status_callback,
|
||||
selector,
|
||||
source_context,
|
||||
_countdown_attempts + 1,
|
||||
)
|
||||
|
||||
link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
|
||||
logger.warning("No download URL found. First 10 links: %s", link_texts)
|
||||
# A bypassed page with no AA download links often means the network served a wrong
|
||||
# page (e.g. an ISP block page) instead of Anna's Archive. Probe for DNS interference
|
||||
# so we can give the user an actionable hint instead of a generic failure.
|
||||
host = urlparse(link).hostname or ""
|
||||
if host:
|
||||
network.note_possible_dns_interference(host)
|
||||
return ""
|
||||
|
||||
|
||||
@@ -1646,7 +1937,6 @@ class DirectDownloadSource(ReleaseSource):
|
||||
except Exception:
|
||||
logger.exception("Search error")
|
||||
|
||||
logger.info("Found %s releases via title+author", len(all_results))
|
||||
return [_browse_record_to_release(record) for record in all_results]
|
||||
|
||||
def is_available(self) -> bool:
|
||||
@@ -1769,7 +2059,14 @@ class DirectDownloadHandler(DownloadHandler):
|
||||
return None
|
||||
|
||||
if not success_url:
|
||||
status_callback("error", "All download sources failed")
|
||||
if network.dns_interference_detected():
|
||||
status_callback(
|
||||
"error",
|
||||
"All sources failed - your network/ISP appears to be blocking "
|
||||
"Anna's Archive. Enable DNS-over-HTTPS in settings.",
|
||||
)
|
||||
else:
|
||||
status_callback("error", "All download sources failed")
|
||||
return None
|
||||
|
||||
# Return temp path - orchestrator handles post-processing (archive extraction, ingest)
|
||||
|
||||
@@ -13,7 +13,6 @@ from typing import Any
|
||||
|
||||
from shelfmark.config import env
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.release_sources import Release, ReleaseProtocol
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -56,12 +55,6 @@ def _coerce_timestamp(value: object) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _generate_cache_key(provider: str, provider_id: str, content_type: str | None = None) -> str:
|
||||
"""Generate a cache key from provider, provider_id, and content type."""
|
||||
normalized_content_type = "audiobook" if check_audiobook(content_type) else "ebook"
|
||||
return f"{provider}:{provider_id}:{normalized_content_type}"
|
||||
|
||||
|
||||
def _load_cache() -> dict[str, Any]:
|
||||
"""Load cache from disk."""
|
||||
try:
|
||||
@@ -103,17 +96,17 @@ def _dict_to_release(data: dict[str, Any]) -> Release:
|
||||
|
||||
|
||||
def get_cached_results(
|
||||
provider: str,
|
||||
provider_id: str,
|
||||
content_type: str | None = None,
|
||||
cache_key: str,
|
||||
ttl_seconds: int | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Get cached search results for a book.
|
||||
"""Get the cached IRC answer for a query identity (server:channel:query).
|
||||
|
||||
The cache stores the whole answer (releases for all content types) under the query
|
||||
identity, so it is not isolated by book or content type. Callers filter by content
|
||||
type after reading.
|
||||
|
||||
Args:
|
||||
provider: Metadata provider name (e.g., "hardcover", "openlibrary")
|
||||
provider_id: Book ID in the provider's system
|
||||
content_type: Search content type for cache isolation
|
||||
cache_key: Query identity (e.g. "server:channel:query")
|
||||
ttl_seconds: Cache TTL in seconds (from settings)
|
||||
|
||||
Returns:
|
||||
@@ -127,8 +120,6 @@ def get_cached_results(
|
||||
ttl_value = config.get("IRC_CACHE_TTL", DEFAULT_CACHE_TTL)
|
||||
ttl_seconds = _coerce_cache_ttl(ttl_value, DEFAULT_CACHE_TTL)
|
||||
|
||||
cache_key = _generate_cache_key(provider, provider_id, content_type)
|
||||
|
||||
with _cache_lock:
|
||||
cache = _load_cache()
|
||||
entry = cache.get("entries", {}).get(cache_key)
|
||||
@@ -141,10 +132,9 @@ def get_cached_results(
|
||||
age = time.time() - cached_at
|
||||
|
||||
if ttl_seconds != 0 and age > ttl_seconds:
|
||||
title = entry.get("title", cache_key)
|
||||
logger.debug(
|
||||
"IRC cache expired for '%s' (age: %.0fs > TTL: %ss)",
|
||||
title,
|
||||
entry.get("title", cache_key),
|
||||
age,
|
||||
ttl_seconds,
|
||||
)
|
||||
@@ -153,44 +143,36 @@ def get_cached_results(
|
||||
|
||||
# Convert dicts back to Release objects
|
||||
releases = [_dict_to_release(r) for r in entry.get("releases", [])]
|
||||
online_servers = entry.get("online_servers", [])
|
||||
title = entry.get("title", "")
|
||||
|
||||
logger.info(
|
||||
"IRC cache hit for '%s' (%s releases, age: %.0fs)",
|
||||
title,
|
||||
entry.get("title", ""),
|
||||
len(releases),
|
||||
age,
|
||||
)
|
||||
|
||||
return {
|
||||
"releases": releases,
|
||||
"online_servers": online_servers,
|
||||
"online_servers": entry.get("online_servers", []),
|
||||
"cached_at": cached_at,
|
||||
}
|
||||
|
||||
|
||||
def cache_results(
|
||||
provider: str,
|
||||
provider_id: str,
|
||||
cache_key: str,
|
||||
title: str,
|
||||
releases: list[Release],
|
||||
content_type: str | None = None,
|
||||
online_servers: list[str] | None = None,
|
||||
) -> None:
|
||||
"""Cache search results for a book.
|
||||
"""Cache the whole IRC answer for a query identity.
|
||||
|
||||
Args:
|
||||
provider: Metadata provider name
|
||||
provider_id: Book ID in the provider's system
|
||||
title: Book title (for logging/display)
|
||||
releases: List of Release objects from search
|
||||
content_type: Search content type for cache isolation
|
||||
cache_key: Query identity (e.g. "server:channel:query")
|
||||
title: Query text (for logging/display)
|
||||
releases: All Release objects from the search (every content type)
|
||||
online_servers: List of online server nicks (optional)
|
||||
|
||||
"""
|
||||
cache_key = _generate_cache_key(provider, provider_id, content_type)
|
||||
|
||||
with _cache_lock:
|
||||
cache = _load_cache()
|
||||
|
||||
@@ -198,9 +180,6 @@ def cache_results(
|
||||
cache["entries"] = {}
|
||||
|
||||
cache["entries"][cache_key] = {
|
||||
"provider": provider,
|
||||
"provider_id": provider_id,
|
||||
"content_type": "audiobook" if check_audiobook(content_type) else "ebook",
|
||||
"title": title,
|
||||
"releases": [_release_to_dict(r) for r in releases],
|
||||
"online_servers": list(online_servers) if online_servers else [],
|
||||
@@ -211,27 +190,23 @@ def cache_results(
|
||||
logger.info("Cached %s IRC releases for '%s'", len(releases), title)
|
||||
|
||||
|
||||
def invalidate_cache(provider: str, provider_id: str, content_type: str | None = None) -> bool:
|
||||
def invalidate_cache(cache_key: str) -> bool:
|
||||
"""Remove a specific entry from the cache.
|
||||
|
||||
Args:
|
||||
provider: Metadata provider name
|
||||
provider_id: Book ID in the provider's system
|
||||
content_type: Search content type for cache isolation
|
||||
cache_key: Query identity to remove
|
||||
|
||||
Returns:
|
||||
True if entry was found and removed
|
||||
|
||||
"""
|
||||
cache_key = _generate_cache_key(provider, provider_id, content_type)
|
||||
|
||||
with _cache_lock:
|
||||
cache = _load_cache()
|
||||
entry = cache.get("entries", {}).get(cache_key)
|
||||
title = entry.get("title", cache_key) if entry else cache_key
|
||||
entries = cache.get("entries", {})
|
||||
|
||||
if cache_key in cache.get("entries", {}):
|
||||
del cache["entries"][cache_key]
|
||||
if cache_key in entries:
|
||||
title = entries[cache_key].get("title", cache_key)
|
||||
del entries[cache_key]
|
||||
_save_cache(cache)
|
||||
logger.info("Invalidated IRC cache for '%s'", title)
|
||||
return True
|
||||
|
||||
@@ -88,7 +88,11 @@ def irc_settings() -> list[SettingsField]:
|
||||
key="IRC_SEARCH_BOT",
|
||||
label="Search bot",
|
||||
placeholder="e.g. search",
|
||||
description="The search bot to query for results",
|
||||
description=(
|
||||
"The search bot to address queries to (required). Searches are sent as "
|
||||
'"@<bot> <query>".'
|
||||
),
|
||||
required=True,
|
||||
env_supported=True,
|
||||
),
|
||||
HeadingField(
|
||||
|
||||
@@ -15,6 +15,7 @@ if TYPE_CHECKING:
|
||||
from shelfmark.api.websocket import ws_manager
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import is_audiobook
|
||||
from shelfmark.release_sources import (
|
||||
ColumnColorHint,
|
||||
ColumnRenderType,
|
||||
@@ -88,6 +89,17 @@ def _emit_status(message: str, phase: str = "searching") -> None:
|
||||
MIN_SEARCH_INTERVAL = 15.0
|
||||
_last_search_time: float = 0
|
||||
|
||||
# Anti-spam budget: the exact same message may only be posted to the channel a limited
|
||||
# number of times within a rolling window. This stops a retry/refresh loop from flooding
|
||||
# the channel with the same line over and over, while still allowing a few genuine retries
|
||||
# (a search that came back empty can be tried again, and Refresh works until the budget runs
|
||||
# out). Normal use never hits this: successful searches are served from the result cache
|
||||
# without re-posting at all.
|
||||
MAX_IDENTICAL_SENDS = 3
|
||||
IDENTICAL_SEND_WINDOW_SECONDS = 24 * 60 * 60 # 24 hours
|
||||
# message-send-key -> timestamps of recent posts of that exact message
|
||||
_recent_message_sends: dict[str, list[float]] = {}
|
||||
|
||||
|
||||
def _enforce_rate_limit() -> None:
|
||||
"""Ensure minimum time between searches."""
|
||||
@@ -102,6 +114,36 @@ def _enforce_rate_limit() -> None:
|
||||
_last_search_time = time.time()
|
||||
|
||||
|
||||
def _query_identity(server: str, channel: str, query: str) -> str:
|
||||
"""Stable identity for a query on a given IRC server-channel.
|
||||
|
||||
Used as BOTH the result-cache key and the per-query send-counter key, so the same
|
||||
query shares one cached answer and one send budget regardless of which book or
|
||||
content type triggered it.
|
||||
"""
|
||||
return f"{server.casefold()}:{channel.casefold()}:{query.strip().casefold()}"
|
||||
|
||||
|
||||
def _recent_send_count(key: str) -> int:
|
||||
"""Number of times this exact message was posted within the rolling window."""
|
||||
cutoff = time.time() - IDENTICAL_SEND_WINDOW_SECONDS
|
||||
timestamps = [ts for ts in _recent_message_sends.get(key, []) if ts > cutoff]
|
||||
if timestamps:
|
||||
_recent_message_sends[key] = timestamps
|
||||
else:
|
||||
_recent_message_sends.pop(key, None)
|
||||
return len(timestamps)
|
||||
|
||||
|
||||
def _record_message_sent(key: str) -> None:
|
||||
"""Record that an exact message was just posted to the channel."""
|
||||
now = time.time()
|
||||
cutoff = now - IDENTICAL_SEND_WINDOW_SECONDS
|
||||
timestamps = [ts for ts in _recent_message_sends.get(key, []) if ts > cutoff]
|
||||
timestamps.append(now)
|
||||
_recent_message_sends[key] = timestamps
|
||||
|
||||
|
||||
@register_source("irc")
|
||||
class IRCReleaseSource(ReleaseSource):
|
||||
"""Search IRC channels for ebook and audiobook releases."""
|
||||
@@ -117,11 +159,16 @@ class IRCReleaseSource(ReleaseSource):
|
||||
self._online_servers: set[str] | None = None
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if IRC is configured (server, channel, and nick are set)."""
|
||||
"""Check if IRC is configured (server, channel, nick, and search bot are set).
|
||||
|
||||
The search bot is required: without it we would post bare queries straight
|
||||
to the channel, which reads as spam and gets the nick banned.
|
||||
"""
|
||||
server = _config_text("IRC_SERVER")
|
||||
channel = _config_text("IRC_CHANNEL")
|
||||
nick = _config_text("IRC_NICK")
|
||||
return bool(server and channel and nick)
|
||||
search_bot = _config_text("IRC_SEARCH_BOT")
|
||||
return bool(server and channel and nick and search_bot)
|
||||
|
||||
def get_column_config(self) -> ReleaseColumnConfig:
|
||||
"""Configure UI columns for IRC results."""
|
||||
@@ -179,25 +226,12 @@ class IRCReleaseSource(ReleaseSource):
|
||||
logger.debug("IRC source is disabled, skipping search")
|
||||
return []
|
||||
|
||||
# Check cache first (unless expand_search/refresh is requested)
|
||||
if not expand_search:
|
||||
cached = get_cached_results(book.provider, book.provider_id, content_type=content_type)
|
||||
if cached:
|
||||
_emit_status("Using cached results", phase="complete")
|
||||
self._online_servers = set(cached.get("online_servers", []))
|
||||
return cached["releases"]
|
||||
|
||||
# Build search query
|
||||
query = plan.primary_query or self._build_query(book)
|
||||
if not query:
|
||||
logger.warning("No search query could be built")
|
||||
return []
|
||||
|
||||
logger.info("IRC search: %s", query)
|
||||
|
||||
# Enforce rate limit
|
||||
_enforce_rate_limit()
|
||||
|
||||
# Get IRC settings
|
||||
server = _config_text("IRC_SERVER")
|
||||
port = _config_port("IRC_PORT", 6697)
|
||||
@@ -206,6 +240,54 @@ class IRCReleaseSource(ReleaseSource):
|
||||
nick = _config_text("IRC_NICK")
|
||||
search_bot = _config_text("IRC_SEARCH_BOT")
|
||||
|
||||
# Never post an unaddressed query to the channel. A bare book title looks like
|
||||
# spam to everyone else in the channel and gets the nick banned. Searches must
|
||||
# be addressed to a search bot ("@<bot> <query>").
|
||||
if not search_bot:
|
||||
logger.warning(
|
||||
"IRC search bot not configured; refusing to post unaddressed query to channel"
|
||||
)
|
||||
_emit_status("IRC search bot not configured", phase="error")
|
||||
return []
|
||||
|
||||
# One identity per query on this server-channel. The result cache and the send
|
||||
# counter are both keyed on it: the SAME query shares one cached answer and one
|
||||
# send budget regardless of which book/content type triggered it, while different
|
||||
# queries are independent (searching 100 different books posts 100 messages).
|
||||
requested = "audiobook" if is_audiobook(content_type) else "ebook"
|
||||
query_key = _query_identity(server, channel, query)
|
||||
|
||||
# Serve the cached whole answer for an identical query (unless this is a refresh).
|
||||
if not expand_search:
|
||||
cached = get_cached_results(query_key)
|
||||
if cached:
|
||||
_emit_status("Using cached results", phase="complete")
|
||||
self._online_servers = set(cached.get("online_servers", []))
|
||||
return self._filter_by_content_type(cached["releases"], requested)
|
||||
|
||||
# Anti-spam cap: the exact same query may only be POSTED a limited number of times
|
||||
# per window, even via refresh. Beyond that, serve whatever is cached rather than
|
||||
# re-posting the identical message to the channel.
|
||||
if _recent_send_count(query_key) >= MAX_IDENTICAL_SENDS:
|
||||
logger.info(
|
||||
"IRC query hit %s-send limit in window, not re-posting: %s",
|
||||
MAX_IDENTICAL_SENDS,
|
||||
query,
|
||||
)
|
||||
_emit_status(
|
||||
"Search limit reached for this query — showing latest results", phase="complete"
|
||||
)
|
||||
cached = get_cached_results(query_key)
|
||||
if cached:
|
||||
self._online_servers = set(cached.get("online_servers", []))
|
||||
return self._filter_by_content_type(cached["releases"], requested)
|
||||
return []
|
||||
|
||||
logger.info("IRC search: %s", query)
|
||||
|
||||
# Enforce rate limit
|
||||
_enforce_rate_limit()
|
||||
|
||||
client = None
|
||||
try:
|
||||
# Get or reuse IRC connection
|
||||
@@ -221,28 +303,27 @@ class IRCReleaseSource(ReleaseSource):
|
||||
# Capture online servers (elevated users in channel)
|
||||
self._online_servers = client.online_servers
|
||||
|
||||
# Send search request
|
||||
search_msg = f"@{search_bot} {query}" if search_bot else query
|
||||
client.send_message(f"#{channel}", search_msg)
|
||||
# Send search request (always addressed to the search bot, never bare)
|
||||
client.send_message(f"#{channel}", f"@{search_bot} {query}")
|
||||
_record_message_sent(query_key)
|
||||
|
||||
# Wait for results DCC - this is the long wait
|
||||
# Wait for results DCC - this is the long wait.
|
||||
# Don't restrict the sender to the trigger bot's nick: many channels answer an
|
||||
# "@search" from a differently-named results bot. The DCC endpoint/filename are
|
||||
# still validated, and wait_for_dcc falls back to the channel's server list.
|
||||
_emit_status(f"Connected to #{channel} - Waiting for results...", phase="searching")
|
||||
wait_kwargs = {"expected_senders": {search_bot}} if search_bot else {}
|
||||
offer = client.wait_for_dcc(timeout=60.0, result_type=True, **wait_kwargs)
|
||||
offer = client.wait_for_dcc(timeout=60.0, result_type=True)
|
||||
|
||||
online_servers = list(self._online_servers) if self._online_servers else None
|
||||
|
||||
if not offer:
|
||||
logger.info("No search results received")
|
||||
_emit_status("No results found", phase="complete")
|
||||
# Release connection for reuse (don't close it)
|
||||
connection_manager.release_connection(client)
|
||||
# Cache empty result to avoid repeated failed searches
|
||||
cache_results(
|
||||
book.provider,
|
||||
book.provider_id,
|
||||
book.title,
|
||||
[],
|
||||
content_type=content_type,
|
||||
online_servers=list(self._online_servers) if self._online_servers else None,
|
||||
)
|
||||
# Cache the (empty) answer under the query identity so an identical query
|
||||
# is served from cache instead of re-posting.
|
||||
cache_results(query_key, query, [], online_servers=online_servers)
|
||||
return []
|
||||
|
||||
# Download results file
|
||||
@@ -260,19 +341,22 @@ class IRCReleaseSource(ReleaseSource):
|
||||
# Release connection for reuse (don't close it)
|
||||
connection_manager.release_connection(client)
|
||||
|
||||
# Convert to Release objects
|
||||
results = parse_results_file(content, content_type=content_type)
|
||||
releases = self._convert_to_releases(results, content_type=content_type)
|
||||
|
||||
# Cache results
|
||||
cache_results(
|
||||
book.provider,
|
||||
book.provider_id,
|
||||
book.title,
|
||||
releases,
|
||||
content_type=content_type,
|
||||
online_servers=list(self._online_servers) if self._online_servers else None,
|
||||
# A single "@search" returns one file containing every format. Parse the whole
|
||||
# answer (both ebooks and audiobooks) and cache it under the query identity, so
|
||||
# requesting the other content type is served from cache without re-posting.
|
||||
ebook_releases = self._convert_to_releases(
|
||||
parse_results_file(content, content_type="ebook"), content_type="ebook"
|
||||
)
|
||||
audiobook_releases = self._convert_to_releases(
|
||||
parse_results_file(content, content_type="audiobook"), content_type="audiobook"
|
||||
)
|
||||
cache_results(
|
||||
query_key,
|
||||
query,
|
||||
ebook_releases + audiobook_releases,
|
||||
online_servers=online_servers,
|
||||
)
|
||||
releases = audiobook_releases if requested == "audiobook" else ebook_releases
|
||||
|
||||
except DCCError as e:
|
||||
logger.exception("DCC error during search")
|
||||
@@ -390,6 +474,15 @@ class IRCReleaseSource(ReleaseSource):
|
||||
|
||||
return releases
|
||||
|
||||
@staticmethod
|
||||
def _filter_by_content_type(releases: list[Release], requested: str) -> list[Release]:
|
||||
"""Pick the requested content type out of a cached whole answer.
|
||||
|
||||
The cache stores releases for every content type under one query identity; each
|
||||
release is tagged with its content type (defaulting to ebook when missing).
|
||||
"""
|
||||
return [release for release in releases if (release.content_type or "ebook") == requested]
|
||||
|
||||
@staticmethod
|
||||
def _parse_size(size_str: str) -> int | None:
|
||||
"""Parse human-readable size (e.g., '1.2MB', '500K') to bytes."""
|
||||
|
||||
@@ -124,10 +124,12 @@ class ProwlarrClient:
|
||||
msg = f"Invalid JSON response: {e}"
|
||||
raise ValueError(msg) from e
|
||||
except requests.exceptions.HTTPError as e:
|
||||
status_code = e.response.status_code if e.response is not None else "unknown"
|
||||
reason = e.response.reason if e.response is not None else "unknown"
|
||||
logger.exception(
|
||||
"Prowlarr API HTTP error: %s %s",
|
||||
e.response.status_code,
|
||||
e.response.reason,
|
||||
status_code,
|
||||
reason,
|
||||
)
|
||||
raise
|
||||
except requests.exceptions.RequestException:
|
||||
|
||||
@@ -7,5 +7,5 @@
|
||||
|
||||
// This script is intentionally loaded from index.html as a classic script,
|
||||
// so we need to declare it as an entry point manually.
|
||||
"entry": ["public/theme-init.js"]
|
||||
"entry": ["public/theme-init.js"],
|
||||
}
|
||||
|
||||
Generated
+821
-675
File diff suppressed because it is too large
Load Diff
+13
-13
@@ -17,24 +17,24 @@
|
||||
"test:unit": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tailwindcss/vite": "^4.2.4",
|
||||
"react": "^19.2.6",
|
||||
"react-dom": "^19.2.6",
|
||||
"react-router-dom": "^7.15.0",
|
||||
"@tailwindcss/vite": "^4.3.2",
|
||||
"react": "^19.2.7",
|
||||
"react-dom": "^19.2.7",
|
||||
"react-router-dom": "^7.18.1",
|
||||
"socket.io-client": "^4.7.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.6.2",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/node": "^26.0.1",
|
||||
"@types/react": "^19.2.17",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^6.0.1",
|
||||
"knip": "^6.12.1",
|
||||
"oxfmt": "^0.48.0",
|
||||
"oxlint": "^1.63.0",
|
||||
"oxlint-tsgolint": "^0.22.1",
|
||||
"@vitejs/plugin-react": "^6.0.3",
|
||||
"knip": "^6.23.0",
|
||||
"oxfmt": "^0.57.0",
|
||||
"oxlint": "^1.72.0",
|
||||
"oxlint-tsgolint": "^0.23.0",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^6.0.3",
|
||||
"vite": "^8.0.11",
|
||||
"vitest": "^4.1.5"
|
||||
"vite": "^8.1.0",
|
||||
"vitest": "^4.1.9"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -766,7 +766,8 @@ function App() {
|
||||
(cfg.show_combined_selector ?? true) &&
|
||||
getDefaultMode('ebook') !== 'blocked' &&
|
||||
getDefaultMode('audiobook') !== 'blocked';
|
||||
const nextEffectiveCombinedMode = combinedMode && nextCombinedModeAllowed;
|
||||
const nextEffectiveCombinedMode =
|
||||
nextCombinedModeAllowed && (combinedMode || cfg.force_combined_search);
|
||||
const activeConfiguredProvider =
|
||||
nextEffectiveCombinedMode && metadataProviderState.configured_provider_combined
|
||||
? metadataProviderState.configured_provider_combined
|
||||
@@ -864,7 +865,8 @@ function App() {
|
||||
const audiobookMode = getDefaultMode('audiobook');
|
||||
return ebookMode !== 'blocked' && audiobookMode !== 'blocked';
|
||||
}, [effectiveSearchMode, config?.show_combined_selector, getDefaultMode]);
|
||||
const effectiveCombinedMode = combinedMode && combinedModeAllowed;
|
||||
const combinedModeLocked = combinedModeAllowed && config?.force_combined_search === true;
|
||||
const effectiveCombinedMode = combinedModeAllowed && (combinedMode || combinedModeLocked);
|
||||
const effectiveCombinedState = effectiveCombinedMode ? combinedState : null;
|
||||
|
||||
const defaultMetadataProviderForContentType =
|
||||
@@ -1489,7 +1491,7 @@ function App() {
|
||||
const handleCancel = async (id: string) => {
|
||||
try {
|
||||
await cancelDownload(id);
|
||||
await fetchStatus();
|
||||
await Promise.all([fetchStatus(), refreshActivitySnapshot()]);
|
||||
} catch (error) {
|
||||
console.error('Cancel failed:', error);
|
||||
showToast('Failed to cancel/clear download', 'error');
|
||||
@@ -2418,6 +2420,7 @@ function App() {
|
||||
onContentTypeChange={setContentType}
|
||||
allowedContentTypes={allowedContentTypes}
|
||||
combinedMode={effectiveCombinedMode}
|
||||
combinedModeLocked={combinedModeLocked}
|
||||
onCombinedModeChange={combinedModeAllowed ? setCombinedMode : undefined}
|
||||
queryTargets={queryTargets}
|
||||
activeQueryTarget={effectiveActiveQueryTarget}
|
||||
@@ -2499,6 +2502,7 @@ function App() {
|
||||
onContentTypeChange={setContentType}
|
||||
allowedContentTypes={allowedContentTypes}
|
||||
combinedMode={effectiveCombinedMode}
|
||||
combinedModeLocked={combinedModeLocked}
|
||||
onCombinedModeChange={combinedModeAllowed ? setCombinedMode : undefined}
|
||||
activeQueryField={activeQueryField}
|
||||
searchMode={effectiveSearchMode}
|
||||
@@ -2774,10 +2778,13 @@ function App() {
|
||||
parsedParams={parsedParams}
|
||||
config={config}
|
||||
contentType={contentType}
|
||||
combinedMode={combinedMode}
|
||||
combinedModeAllowed={combinedModeAllowed}
|
||||
advancedFilters={advancedFilters}
|
||||
resolvedMetadataDefaultSort={resolvedMetadataDefaultSort}
|
||||
resolvedMetadataSortOptions={resolvedMetadataSortOptions}
|
||||
setContentType={setContentType}
|
||||
setCombinedMode={setCombinedMode}
|
||||
setSearchInput={setSearchInput}
|
||||
setAdvancedFilters={setAdvancedFilters}
|
||||
setShowAdvanced={setShowAdvanced}
|
||||
|
||||
@@ -58,6 +58,8 @@ const SEARCH_MODE_OPTIONS = [
|
||||
},
|
||||
];
|
||||
|
||||
const EMPTY_PROVIDERS: MetadataProviderSummary[] = [];
|
||||
|
||||
export const AdvancedFilters = ({
|
||||
visible,
|
||||
bookLanguages,
|
||||
@@ -68,7 +70,7 @@ export const AdvancedFilters = ({
|
||||
renderWrapper,
|
||||
searchMode,
|
||||
onSearchModeChange,
|
||||
metadataProviders = [],
|
||||
metadataProviders = EMPTY_PROVIDERS,
|
||||
activeMetadataProvider,
|
||||
onMetadataProviderChange,
|
||||
contentType = 'ebook',
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { ReactNode } from 'react';
|
||||
import { useCallback, useLayoutEffect, useRef, useState } from 'react';
|
||||
import { useCallback, useId, useLayoutEffect, useRef, useState } from 'react';
|
||||
import { createPortal } from 'react-dom';
|
||||
|
||||
import { useDismiss } from '../hooks/useDismiss';
|
||||
@@ -55,6 +55,7 @@ export const Dropdown = ({
|
||||
onOpenChange,
|
||||
}: DropdownProps) => {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const dropdownId = useId();
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const triggerRef = useRef<HTMLDivElement>(null);
|
||||
const panelRef = useRef<HTMLDivElement>(null);
|
||||
@@ -240,8 +241,8 @@ export const Dropdown = ({
|
||||
<div className={widthClassName} ref={containerRef}>
|
||||
{label && (
|
||||
<label
|
||||
className="mb-1.5 block text-xs font-medium text-gray-500 dark:text-gray-400"
|
||||
onClick={toggleOpen}
|
||||
htmlFor={dropdownId}
|
||||
className="mb-1.5 block cursor-pointer text-xs font-medium text-gray-500 dark:text-gray-400"
|
||||
>
|
||||
{label}
|
||||
</label>
|
||||
@@ -251,6 +252,7 @@ export const Dropdown = ({
|
||||
renderTrigger({ isOpen, toggle: toggleOpen })
|
||||
) : (
|
||||
<button
|
||||
id={dropdownId}
|
||||
type="button"
|
||||
onClick={toggleOpen}
|
||||
disabled={disabled}
|
||||
|
||||
@@ -147,6 +147,7 @@ export const DropdownList = ({
|
||||
type="checkbox"
|
||||
checked={selectedValues.includes(option.value)}
|
||||
readOnly
|
||||
aria-label={option.label}
|
||||
className="pointer-events-none h-4 w-4 rounded-sm border-gray-300 text-sky-600 focus:ring-sky-500"
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -62,6 +62,7 @@ interface HeaderProps {
|
||||
onContentTypeChange?: (type: ContentType) => void;
|
||||
allowedContentTypes?: ContentType[];
|
||||
combinedMode?: boolean;
|
||||
combinedModeLocked?: boolean;
|
||||
onCombinedModeChange?: (enabled: boolean) => void;
|
||||
queryTargets?: QueryTargetOption[];
|
||||
activeQueryTarget?: string;
|
||||
@@ -79,6 +80,15 @@ const applyTheme = (preference: string): void => {
|
||||
document.documentElement.style.colorScheme = effective;
|
||||
};
|
||||
|
||||
const DEFAULT_STATUS_COUNTS: ActivityStatusCounts = {
|
||||
ongoing: 0,
|
||||
completed: 0,
|
||||
errored: 0,
|
||||
pendingRequests: 0,
|
||||
};
|
||||
const EMPTY_ADMIN_USERS: ActingAsUserSelection[] = [];
|
||||
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
|
||||
|
||||
export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
(
|
||||
{
|
||||
@@ -98,7 +108,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
onSettingsClick,
|
||||
isAdmin = false,
|
||||
canAccessSettings,
|
||||
statusCounts = { ongoing: 0, completed: 0, errored: 0, pendingRequests: 0 },
|
||||
statusCounts = DEFAULT_STATUS_COUNTS,
|
||||
onLogoClick,
|
||||
authRequired = false,
|
||||
isAuthenticated = false,
|
||||
@@ -106,7 +116,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
displayName,
|
||||
actingAsUser = null,
|
||||
onActingAsUserChange,
|
||||
adminUsers = [],
|
||||
adminUsers = EMPTY_ADMIN_USERS,
|
||||
isAdminUsersLoading = false,
|
||||
adminUsersError = null,
|
||||
hasLoadedAdminUsers = false,
|
||||
@@ -118,8 +128,9 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
onContentTypeChange,
|
||||
allowedContentTypes,
|
||||
combinedMode,
|
||||
combinedModeLocked,
|
||||
onCombinedModeChange,
|
||||
queryTargets = [],
|
||||
queryTargets = EMPTY_QUERY_TARGETS,
|
||||
activeQueryTarget = 'general',
|
||||
onQueryTargetChange,
|
||||
activeQueryField = null,
|
||||
@@ -305,8 +316,8 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
// Determine if we should show icons only (both URLs configured)
|
||||
const showIconsOnly = Boolean(calibreWebUrl && audiobookLibraryUrl);
|
||||
|
||||
// Icon buttons component - reused for both states
|
||||
const IconButtons = () => (
|
||||
// Icon buttons - reused for both states
|
||||
const iconButtonsNode = (
|
||||
<div className="flex items-center gap-2">
|
||||
{/* Book Library Button */}
|
||||
{calibreWebUrl && (
|
||||
@@ -579,6 +590,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
onClick={handleLogout}
|
||||
className="hover-action shrink-0 rounded-full p-2 text-red-600 transition-colors dark:text-red-400"
|
||||
title="Sign Out"
|
||||
aria-label="Sign Out"
|
||||
>
|
||||
<svg
|
||||
className="h-5 w-5"
|
||||
@@ -674,7 +686,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
/>
|
||||
))}
|
||||
|
||||
<IconButtons />
|
||||
{iconButtonsNode}
|
||||
</div>
|
||||
|
||||
{/* Search bar - appear second on mobile (below logo+icons), first on desktop (left side) */}
|
||||
@@ -711,6 +723,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
onContentTypeChange={onContentTypeChange}
|
||||
allowedContentTypes={allowedContentTypes}
|
||||
combinedMode={combinedMode}
|
||||
combinedModeLocked={combinedModeLocked}
|
||||
onCombinedModeChange={onCombinedModeChange}
|
||||
queryTargets={queryTargets}
|
||||
activeQueryTarget={activeQueryTarget}
|
||||
@@ -723,9 +736,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(
|
||||
|
||||
{/* When search is NOT active: show icon buttons only on the right */}
|
||||
{!showSearch && (
|
||||
<div className="flex min-h-[48px] items-center justify-end">
|
||||
<IconButtons />
|
||||
</div>
|
||||
<div className="flex min-h-[48px] items-center justify-end">{iconButtonsNode}</div>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -101,6 +101,7 @@ const PasswordLoginForm = ({
|
||||
type="text"
|
||||
id="username"
|
||||
name="username"
|
||||
aria-label="Username"
|
||||
autoComplete="username"
|
||||
autoCapitalize="none"
|
||||
autoCorrect="off"
|
||||
@@ -131,6 +132,7 @@ const PasswordLoginForm = ({
|
||||
type={showPassword ? 'text' : 'password'}
|
||||
id="password"
|
||||
name="password"
|
||||
aria-label="Password"
|
||||
autoComplete="current-password"
|
||||
autoCapitalize="none"
|
||||
autoCorrect="off"
|
||||
@@ -168,6 +170,7 @@ const PasswordLoginForm = ({
|
||||
checked={rememberMe}
|
||||
onChange={(event) => setRememberMe(event.target.checked)}
|
||||
disabled={isLoading}
|
||||
aria-label="Remember me for 7 days"
|
||||
className="h-4 w-4 rounded-sm accent-sky-900 focus:ring-2 focus:ring-sky-500 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
style={{ borderColor: 'var(--border-color)' }}
|
||||
/>
|
||||
|
||||
@@ -729,6 +729,8 @@ function ErrorState({ message }: { message: string }) {
|
||||
);
|
||||
}
|
||||
|
||||
const EMPTY_SUPPORTED_AUDIOBOOK_FORMATS: string[] = [];
|
||||
|
||||
const ReleaseModalSession = ({
|
||||
book,
|
||||
onClose,
|
||||
@@ -737,7 +739,7 @@ const ReleaseModalSession = ({
|
||||
onRequestBook,
|
||||
getPolicyModeForSource,
|
||||
supportedFormats,
|
||||
supportedAudiobookFormats = [],
|
||||
supportedAudiobookFormats = EMPTY_SUPPORTED_AUDIOBOOK_FORMATS,
|
||||
contentType,
|
||||
defaultLanguages,
|
||||
bookLanguages,
|
||||
@@ -2094,6 +2096,7 @@ const ReleaseModalSession = ({
|
||||
value={manualQuery}
|
||||
onChange={(e) => setManualQuery(e.target.value)}
|
||||
placeholder="Type a custom search query (overrides all sources)"
|
||||
aria-label="Custom search query"
|
||||
className="w-full rounded-lg border border-(--border-muted) bg-(--bg) px-3 py-2 text-sm text-(--text)"
|
||||
/>
|
||||
<button
|
||||
|
||||
@@ -64,9 +64,11 @@ const getRequestConfirmationSessionKey = (payload: CreateRequestPayload): string
|
||||
].join('|');
|
||||
};
|
||||
|
||||
const EMPTY_PAYLOADS: CreateRequestPayload[] = [];
|
||||
|
||||
export function RequestConfirmationModal({
|
||||
payload,
|
||||
extraPayloads = [],
|
||||
extraPayloads = EMPTY_PAYLOADS,
|
||||
allowNotes,
|
||||
onConfirm,
|
||||
onClose,
|
||||
@@ -89,7 +91,7 @@ export function RequestConfirmationModal({
|
||||
|
||||
function RequestConfirmationModalSession({
|
||||
payload,
|
||||
extraPayloads = [],
|
||||
extraPayloads = EMPTY_PAYLOADS,
|
||||
allowNotes,
|
||||
onConfirm,
|
||||
onClose,
|
||||
@@ -305,6 +307,7 @@ function RequestConfirmationModalSession({
|
||||
</label>
|
||||
<textarea
|
||||
id="request-note"
|
||||
aria-label="Note (optional)"
|
||||
value={note}
|
||||
onChange={(event) => setNote(truncateRequestNote(event.target.value))}
|
||||
maxLength={MAX_REQUEST_NOTE_LENGTH}
|
||||
|
||||
@@ -34,6 +34,7 @@ interface SearchBarProps {
|
||||
onContentTypeChange?: (type: ContentType) => void;
|
||||
allowedContentTypes?: ContentType[];
|
||||
combinedMode?: boolean;
|
||||
combinedModeLocked?: boolean;
|
||||
onCombinedModeChange?: (enabled: boolean) => void;
|
||||
queryTargets?: QueryTargetOption[];
|
||||
activeQueryTarget?: string;
|
||||
@@ -48,6 +49,7 @@ export interface SearchBarHandle {
|
||||
|
||||
const EMPTY_SORT_OPTIONS: SortOption[] = [];
|
||||
const EMPTY_AUTOCOMPLETE_OPTIONS: DynamicFieldOption[] = [];
|
||||
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
|
||||
|
||||
const BookIcon = () => (
|
||||
<svg
|
||||
@@ -180,8 +182,9 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
onContentTypeChange,
|
||||
allowedContentTypes,
|
||||
combinedMode = false,
|
||||
combinedModeLocked = false,
|
||||
onCombinedModeChange,
|
||||
queryTargets = [],
|
||||
queryTargets = EMPTY_QUERY_TARGETS,
|
||||
activeQueryTarget = 'general',
|
||||
onQueryTargetChange,
|
||||
activeQueryField,
|
||||
@@ -518,6 +521,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
type="checkbox"
|
||||
checked={Boolean(value)}
|
||||
onChange={(e) => onChange(e.target.checked)}
|
||||
aria-label={activeQueryField.label}
|
||||
className="h-4 w-4 rounded-sm border-(--border-muted) text-emerald-500 focus:ring-emerald-500/50"
|
||||
/>
|
||||
<span className="truncate text-sm" style={{ color: 'var(--text)' }}>
|
||||
@@ -796,11 +800,19 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
|
||||
/>
|
||||
{combinedModeLocked ? (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M16.5 10.5V6.75a4.5 4.5 0 1 0-9 0v3.75m-.75 11.25h10.5a2.25 2.25 0 0 0 2.25-2.25v-6.75a2.25 2.25 0 0 0-2.25-2.25H6.75a2.25 2.25 0 0 0-2.25 2.25v6.75a2.25 2.25 0 0 0 2.25 2.25Z"
|
||||
/>
|
||||
) : (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
|
||||
/>
|
||||
)}
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -32,6 +32,7 @@ interface SearchSectionProps {
|
||||
onContentTypeChange?: (type: ContentType) => void;
|
||||
allowedContentTypes?: ContentType[];
|
||||
combinedMode?: boolean;
|
||||
combinedModeLocked?: boolean;
|
||||
onCombinedModeChange?: (enabled: boolean) => void;
|
||||
activeQueryField?: MetadataSearchField | null;
|
||||
searchMode: SearchMode;
|
||||
@@ -64,6 +65,7 @@ export const SearchSection = ({
|
||||
onContentTypeChange,
|
||||
allowedContentTypes,
|
||||
combinedMode,
|
||||
combinedModeLocked,
|
||||
onCombinedModeChange,
|
||||
activeQueryField,
|
||||
searchMode,
|
||||
@@ -105,6 +107,7 @@ export const SearchSection = ({
|
||||
onContentTypeChange={onContentTypeChange}
|
||||
allowedContentTypes={allowedContentTypes}
|
||||
combinedMode={combinedMode}
|
||||
combinedModeLocked={combinedModeLocked}
|
||||
onCombinedModeChange={onCombinedModeChange}
|
||||
queryTargets={queryTargets}
|
||||
activeQueryTarget={activeQueryTarget}
|
||||
|
||||
@@ -12,10 +12,13 @@ interface UrlSearchBootstrapMountProps {
|
||||
parsedParams: ParsedUrlSearch;
|
||||
config: AppConfig;
|
||||
contentType: ContentType;
|
||||
combinedMode: boolean;
|
||||
combinedModeAllowed: boolean;
|
||||
advancedFilters: AdvancedFilterState;
|
||||
resolvedMetadataDefaultSort: string;
|
||||
resolvedMetadataSortOptions: SortOption[];
|
||||
setContentType: (value: ContentType) => void;
|
||||
setCombinedMode: (value: boolean) => void;
|
||||
setSearchInput: (value: string) => void;
|
||||
setAdvancedFilters: Dispatch<SetStateAction<AdvancedFilterState>>;
|
||||
setShowAdvanced: (value: boolean) => void;
|
||||
@@ -32,10 +35,13 @@ export const UrlSearchBootstrapMount = ({
|
||||
parsedParams,
|
||||
config,
|
||||
contentType,
|
||||
combinedMode,
|
||||
combinedModeAllowed,
|
||||
advancedFilters,
|
||||
resolvedMetadataDefaultSort,
|
||||
resolvedMetadataSortOptions,
|
||||
setContentType,
|
||||
setCombinedMode,
|
||||
setSearchInput,
|
||||
setAdvancedFilters,
|
||||
setShowAdvanced,
|
||||
@@ -49,11 +55,19 @@ export const UrlSearchBootstrapMount = ({
|
||||
const parsedSearchMode = config.search_mode || 'universal';
|
||||
const urlContentTypeOverride =
|
||||
parsedSearchMode === 'universal' ? parsedParams.contentType : undefined;
|
||||
const urlForcesCombined =
|
||||
parsedSearchMode === 'universal' && parsedParams.combinedMode === true && combinedModeAllowed;
|
||||
|
||||
if (urlContentTypeOverride && urlContentTypeOverride !== contentType) {
|
||||
setContentType(urlContentTypeOverride);
|
||||
}
|
||||
|
||||
if (urlForcesCombined && !combinedMode) {
|
||||
setCombinedMode(true);
|
||||
} else if (urlContentTypeOverride && combinedMode) {
|
||||
setCombinedMode(false);
|
||||
}
|
||||
|
||||
if (!parsedParams.hasSearchParams) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -436,6 +436,7 @@ const RejectInlinePanel = ({
|
||||
Reject request for <span className="opacity-80">{itemTitle || 'Untitled request'}</span>
|
||||
</p>
|
||||
<textarea
|
||||
aria-label="Optional note shown to the user"
|
||||
value={rejectNote}
|
||||
onChange={(event) => setRejectNote(event.target.value.slice(0, MAX_ADMIN_NOTE_LENGTH))}
|
||||
rows={3}
|
||||
|
||||
@@ -225,6 +225,9 @@ const getInitialPinnedPreference = (): boolean => {
|
||||
}
|
||||
};
|
||||
|
||||
const EMPTY_KEYS: string[] = [];
|
||||
const EMPTY_ITEMS: ActivityItem[] = [];
|
||||
|
||||
export const ActivitySidebar = ({
|
||||
isOpen,
|
||||
onClose,
|
||||
@@ -235,8 +238,8 @@ export const ActivitySidebar = ({
|
||||
onRetry,
|
||||
onDownloadDismiss,
|
||||
requestItems,
|
||||
dismissedItemKeys = [],
|
||||
historyItems = [],
|
||||
dismissedItemKeys = EMPTY_KEYS,
|
||||
historyItems = EMPTY_ITEMS,
|
||||
historyLoaded = false,
|
||||
historyHasMore = false,
|
||||
historyLoading = false,
|
||||
|
||||
@@ -67,6 +67,7 @@ export const CardView = ({
|
||||
};
|
||||
|
||||
return (
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
|
||||
<article
|
||||
className="book-card space-between animate-pop-up relative flex h-full w-full flex-col transition-shadow duration-300 max-sm:h-[180px] max-sm:flex-row sm:max-w-[292px] sm:flex-col"
|
||||
style={{
|
||||
|
||||
@@ -70,6 +70,7 @@ export const CompactView = ({
|
||||
};
|
||||
|
||||
return (
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- mouse handlers drive a decorative hover shadow only; no interactive behavior requiring keyboard support
|
||||
<article
|
||||
className="book-card animate-pop-up relative flex! h-[180px]! w-full flex-row! transition-shadow duration-300"
|
||||
style={{
|
||||
|
||||
@@ -102,6 +102,7 @@ export const NamingTemplateField = ({
|
||||
value={value}
|
||||
onChange={(event) => onChange(boundField.key, event.target.value)}
|
||||
placeholder={boundField.placeholder}
|
||||
aria-label={boundField.placeholder || 'Naming Template'}
|
||||
maxLength={boundField.maxLength}
|
||||
disabled={fieldDisabled}
|
||||
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
|
||||
|
||||
@@ -19,6 +19,7 @@ export const NumberField = ({ field, value, onChange, disabled }: NumberFieldPro
|
||||
min={field.min}
|
||||
max={field.max}
|
||||
step={field.step ?? 1}
|
||||
aria-label={field.label || 'Number field'}
|
||||
disabled={isDisabled}
|
||||
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
|
||||
/>
|
||||
|
||||
@@ -21,6 +21,7 @@ export const PasswordField = ({ field, value, onChange, disabled }: PasswordFiel
|
||||
value={value}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
placeholder={field.placeholder}
|
||||
aria-label={field.label || field.placeholder || 'Password field'}
|
||||
disabled={isDisabled}
|
||||
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 pr-10 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
|
||||
/>
|
||||
|
||||
@@ -277,6 +277,7 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
|
||||
checked={Boolean(cellValue)}
|
||||
onChange={(e) => updateCell(rowIndex, col.key, e.target.checked)}
|
||||
disabled={isDisabled}
|
||||
aria-label={`${col.label || col.key} row ${rowIndex + 1}`}
|
||||
className="h-4 w-4 rounded border-gray-300 text-sky-600 focus:ring-sky-500 disabled:cursor-not-allowed disabled:opacity-60"
|
||||
/>
|
||||
</div>
|
||||
@@ -361,6 +362,7 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
|
||||
value={toPrimitiveString(cellValue)}
|
||||
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
|
||||
placeholder={col.placeholder}
|
||||
aria-label={`${col.label || col.key} row ${rowIndex + 1}`}
|
||||
disabled={isDisabled}
|
||||
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
|
||||
/>
|
||||
|
||||
@@ -143,6 +143,7 @@ export const TagListField = ({
|
||||
type="text"
|
||||
value={draft}
|
||||
onChange={(e) => setDraft(e.target.value)}
|
||||
aria-label={field.label || field.placeholder || 'Add item'}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
|
||||
@@ -17,6 +17,7 @@ export const TextField = ({ field, value, onChange, disabled }: TextFieldProps)
|
||||
value={value ?? ''}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
placeholder={field.placeholder}
|
||||
aria-label={field.label || field.placeholder || 'Text field'}
|
||||
maxLength={field.maxLength}
|
||||
disabled={isDisabled}
|
||||
className="w-full rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm transition-colors focus:border-sky-500 focus:ring-2 focus:ring-sky-500/50 focus:outline-hidden disabled:cursor-not-allowed disabled:opacity-60"
|
||||
|
||||
@@ -139,9 +139,11 @@ function formatUserOverrideValue(value: unknown): string {
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_DETAILS: Array<{ userId: number; username: string; value: unknown }> = [];
|
||||
|
||||
const UserOverriddenBadge = ({
|
||||
count,
|
||||
details = [],
|
||||
details = EMPTY_DETAILS,
|
||||
}: {
|
||||
count: number;
|
||||
details?: Array<{ userId: number; username: string; value: unknown }>;
|
||||
|
||||
@@ -60,6 +60,8 @@ const modeDescriptions: Record<RequestPolicyMode, string> = {
|
||||
blocked: 'Downloads and requests are blocked.',
|
||||
};
|
||||
|
||||
const EMPTY_BASE_RULES: RequestPolicyRuleRow[] = [];
|
||||
|
||||
export const RequestPolicyGrid = ({
|
||||
defaultModes,
|
||||
onDefaultModeChange,
|
||||
@@ -67,7 +69,7 @@ export const RequestPolicyGrid = ({
|
||||
defaultModeOverrides,
|
||||
defaultModeDisabled,
|
||||
explicitRules,
|
||||
baseRules = [],
|
||||
baseRules = EMPTY_BASE_RULES,
|
||||
onExplicitRulesChange,
|
||||
sourceCapabilities,
|
||||
rulesDisabled = false,
|
||||
|
||||
@@ -3,6 +3,7 @@ interface ToggleSwitchProps {
|
||||
onChange: (checked: boolean) => void;
|
||||
disabled?: boolean;
|
||||
color?: 'sky' | 'emerald';
|
||||
ariaLabel?: string;
|
||||
}
|
||||
|
||||
const colorClasses = {
|
||||
@@ -15,6 +16,7 @@ export const ToggleSwitch = ({
|
||||
onChange,
|
||||
disabled = false,
|
||||
color = 'sky',
|
||||
ariaLabel = 'Toggle switch',
|
||||
}: ToggleSwitchProps) => {
|
||||
const { active, ring } = colorClasses[color];
|
||||
|
||||
@@ -23,6 +25,7 @@ export const ToggleSwitch = ({
|
||||
type="button"
|
||||
role="switch"
|
||||
aria-checked={checked}
|
||||
aria-label={ariaLabel}
|
||||
onClick={() => !disabled && onChange(!checked)}
|
||||
disabled={disabled}
|
||||
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors duration-200 focus:ring-2 focus:outline-hidden ${ring} disabled:cursor-not-allowed disabled:opacity-60 ${checked ? active : 'bg-gray-300 dark:bg-gray-600'}`}
|
||||
|
||||
@@ -188,6 +188,7 @@ export function Tooltip({
|
||||
isVisible &&
|
||||
coords &&
|
||||
createPortal(
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions -- interactive tooltip stays open while hovered; keyboard users are served by focus handlers on the trigger element
|
||||
<div
|
||||
ref={tooltipRef}
|
||||
role="tooltip"
|
||||
|
||||
@@ -41,7 +41,7 @@ export function useUrlSearch({ enabled }: UseUrlSearchOptions): UseUrlSearchRetu
|
||||
}
|
||||
|
||||
const parsed = parseUrlSearchParams(searchParams);
|
||||
return parsed.hasSearchParams || parsed.contentType ? parsed : null;
|
||||
return parsed.hasSearchParams || parsed.contentType || parsed.combinedMode ? parsed : null;
|
||||
}, [enabled, searchParams]);
|
||||
|
||||
return {
|
||||
|
||||
@@ -42,4 +42,22 @@ describe('parseUrlSearchParams', () => {
|
||||
expect(parsed.hasSearchParams).toBe(false);
|
||||
expect(parsed.contentType).toBe('ebook');
|
||||
});
|
||||
|
||||
it('parses content_type=combined as a combined-mode override', () => {
|
||||
const parsed = parseUrlSearchParams(new URLSearchParams('q=dune&content_type=combined'));
|
||||
|
||||
expect(parsed.searchInput).toBe('dune');
|
||||
expect(parsed.hasSearchParams).toBe(true);
|
||||
expect(parsed.contentType).toBe(undefined);
|
||||
expect(parsed.combinedMode).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps combined-only links from auto-triggering a blank search', () => {
|
||||
const parsed = parseUrlSearchParams(new URLSearchParams('content_type=combined'));
|
||||
|
||||
expect(parsed.searchInput).toBe('');
|
||||
expect(parsed.hasSearchParams).toBe(false);
|
||||
expect(parsed.contentType).toBe(undefined);
|
||||
expect(parsed.combinedMode).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -280,6 +280,7 @@ export interface AppConfig {
|
||||
default_release_source_audiobook?: string; // Default tab in ReleaseModal for audiobooks
|
||||
show_release_source_links: boolean;
|
||||
show_combined_selector: boolean;
|
||||
force_combined_search: boolean;
|
||||
books_output_mode: BooksOutputMode;
|
||||
auto_open_downloads_sidebar: boolean; // Auto-open sidebar when download is queued
|
||||
hardcover_auto_remove_on_download: boolean; // Auto-remove from active Hardcover list on download
|
||||
|
||||
@@ -7,18 +7,24 @@ export interface ParsedUrlSearch {
|
||||
searchInput: string;
|
||||
advancedFilters: Partial<AdvancedFilterState>;
|
||||
contentType?: ContentType;
|
||||
combinedMode?: boolean;
|
||||
hasSearchParams: boolean;
|
||||
}
|
||||
|
||||
const parseContentType = (value: string | null): ContentType | undefined => {
|
||||
const parseContentTypeParam = (
|
||||
value: string | null,
|
||||
): { contentType?: ContentType; combinedMode?: true } => {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
return {};
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === 'ebook' || normalized === 'audiobook') {
|
||||
return normalized;
|
||||
return { contentType: normalized };
|
||||
}
|
||||
return undefined;
|
||||
if (normalized === 'combined') {
|
||||
return { combinedMode: true };
|
||||
}
|
||||
return {};
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -26,21 +32,23 @@ const parseContentType = (value: string | null): ContentType | undefined => {
|
||||
*
|
||||
* Supports both Direct Download and Universal mode parameters.
|
||||
* In Universal mode, query/sort are used for search text, and content_type
|
||||
* is used to select ebook vs audiobook.
|
||||
* selects ebook, audiobook, or combined (search both at once).
|
||||
*
|
||||
* @example
|
||||
* // Direct mode: /?q=harry+potter&author=rowling&format=epub&lang=en
|
||||
* // Universal mode: /?q=dune&sort=popularity
|
||||
* // Universal combined: /?q=dune&content_type=combined
|
||||
*/
|
||||
export function parseUrlSearchParams(searchParams: URLSearchParams): ParsedUrlSearch {
|
||||
const parsedContentType = parseContentType(
|
||||
const contentTypeParam = parseContentTypeParam(
|
||||
searchParams.get('content_type') || searchParams.get('contentType'),
|
||||
);
|
||||
|
||||
const result: ParsedUrlSearch = {
|
||||
searchInput: '',
|
||||
advancedFilters: {},
|
||||
contentType: parsedContentType,
|
||||
contentType: contentTypeParam.contentType,
|
||||
combinedMode: contentTypeParam.combinedMode,
|
||||
hasSearchParams: false,
|
||||
};
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ SAMPLE_DETAIL_HTML = """
|
||||
<table>
|
||||
<tr>
|
||||
<td>Info Hash</td>
|
||||
<td>ABC123DEF456GHI789JKL012MNO345PQR678STU</td>
|
||||
<td>ABC123DEF456789012345678901234567890ABCD</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Tracker 1</td>
|
||||
@@ -83,7 +83,7 @@ DETAIL_HTML_NO_TRACKERS = """
|
||||
<table>
|
||||
<tr>
|
||||
<td>Info Hash</td>
|
||||
<td>ABC123DEF456GHI789JKL012MNO345PQR678STU</td>
|
||||
<td>ABC123DEF456789012345678901234567890ABCD</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
@@ -360,7 +360,7 @@ class TestExtractMagnetLink:
|
||||
|
||||
assert magnet_link is not None
|
||||
assert magnet_link.startswith("magnet:?xt=urn:btih:")
|
||||
assert "ABC123DEF456GHI789JKL012MNO345PQR678STU" in magnet_link
|
||||
assert "ABC123DEF456789012345678901234567890ABCD" in magnet_link
|
||||
assert "udp%3A//tracker.openbittorrent.com%3A80" in magnet_link
|
||||
assert "http%3A//tracker.example.com%3A8080" in magnet_link
|
||||
assert mock_html_get.call_count == 2
|
||||
@@ -395,7 +395,7 @@ class TestExtractMagnetLink:
|
||||
|
||||
assert magnet_link is not None
|
||||
assert magnet_link.startswith("magnet:?xt=urn:btih:")
|
||||
assert "ABC123DEF456GHI789JKL012MNO345PQR678STU" in magnet_link
|
||||
assert "ABC123DEF456789012345678901234567890ABCD" in magnet_link
|
||||
# Should contain default trackers
|
||||
assert "udp%3A//tracker.openbittorrent.com%3A80" in magnet_link
|
||||
|
||||
@@ -441,7 +441,7 @@ class TestExtractMagnetLink:
|
||||
<table>
|
||||
<tr>
|
||||
<td>Info Hash</td>
|
||||
<td>ABC 123 DEF 456</td>
|
||||
<td>ABC 123 DEF 456 789 012 345 678 901 234 567 890 ABC D</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Cluster 7 (audiobook/ABB) parse-contract guards.
|
||||
|
||||
ABB forces ``https://`` for search and detail fetches, so it can't be exercised
|
||||
hermetically in the HTTP e2e docker platform. Its recurring bugs are instead in
|
||||
*parsing*: magnet/info-hash extraction ("Fix ABB magnet parsing", and the
|
||||
qbittorrent hash-length issue #386) and DOM/layout drift. These contract tests
|
||||
feed golden HTML through the real scraper — the same fail-on-drift philosophy as
|
||||
the AA layout-drift guard — and run in normal CI.
|
||||
|
||||
They deliberately cover cases the existing ``test_scraper.py`` does not: info-hash
|
||||
*normalization* (whitespace/case), the in-page magnet *fallback*, and a layout
|
||||
drift that must degrade to an empty result rather than crash.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from unittest.mock import patch
|
||||
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
|
||||
# Detail page where the Info Hash is lowercase and split by whitespace/newlines —
|
||||
# the exact shape that produced malformed magnets / wrong hash lengths (#386).
|
||||
DETAIL_HTML_MESSY_HASH = """
|
||||
<html><body><table>
|
||||
<tr><td>Info Hash</td><td>abc123def456789012345678
|
||||
901234567890abcd</td></tr>
|
||||
<tr><td>Tracker 1</td><td>udp://tracker.openbittorrent.com:80</td></tr>
|
||||
</table></body></html>
|
||||
"""
|
||||
|
||||
# Info Hash cell is junk, but a full magnet link is posted elsewhere on the page.
|
||||
DETAIL_HTML_MAGNET_FALLBACK = """
|
||||
<html><body>
|
||||
<table><tr><td>Info Hash</td><td>n/a</td></tr></table>
|
||||
<p>Mirror: magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=x</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# DOM drift: results are present but the .post / .postTitle structure changed.
|
||||
SEARCH_HTML_LAYOUT_DRIFT = """
|
||||
<html><body>
|
||||
<article class="result-card">
|
||||
<header><a href="/abss/drifted/">Drifted Audiobook - Author</a></header>
|
||||
<span class="lang">English</span>
|
||||
</article>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
def _patch_detail(html: str):
|
||||
return patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
|
||||
return_value=html,
|
||||
)
|
||||
|
||||
|
||||
def test_info_hash_is_normalized_to_canonical_btih() -> None:
|
||||
"""Whitespace/newlines are stripped and the hash upper-cased to a valid
|
||||
40-char btih (regression for #386 / 'Fix ABB magnet parsing')."""
|
||||
with _patch_detail(DETAIL_HTML_MESSY_HASH):
|
||||
magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/x/", "audiobookbay.lu")
|
||||
assert magnet is not None, "messy-but-valid info hash should still yield a magnet"
|
||||
btih = re.search(r"xt=urn:btih:([0-9A-Fa-f]+)", magnet)
|
||||
assert btih is not None, magnet
|
||||
assert btih.group(1) == "ABC123DEF456789012345678901234567890ABCD"
|
||||
assert len(btih.group(1)) == 40
|
||||
assert "tr=" in magnet # tracker carried through
|
||||
|
||||
|
||||
def test_magnet_fallback_when_info_hash_cell_is_junk() -> None:
|
||||
"""When the Info Hash cell is invalid, the scraper recovers the hash from an
|
||||
in-page magnet link rather than failing."""
|
||||
with _patch_detail(DETAIL_HTML_MAGNET_FALLBACK):
|
||||
magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/y/", "audiobookbay.lu")
|
||||
assert magnet is not None
|
||||
assert "btih:1111111111111111111111111111111111111111" in magnet
|
||||
|
||||
|
||||
def test_missing_info_hash_returns_none_not_crash() -> None:
|
||||
"""No hash anywhere -> None (clean failure), never an exception."""
|
||||
with _patch_detail("<html><body><p>nothing here</p></body></html>"):
|
||||
assert (
|
||||
scraper.extract_magnet_link("https://audiobookbay.lu/abss/z/", "audiobookbay.lu")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_search_layout_drift_degrades_to_empty() -> None:
|
||||
"""A changed results DOM yields zero parsed results without raising — the
|
||||
ABB analogue of the AA layout-drift guard."""
|
||||
with (
|
||||
patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
|
||||
return_value=(SEARCH_HTML_LAYOUT_DRIFT, "https://audiobookbay.lu/?s=test"),
|
||||
),
|
||||
patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.config.get",
|
||||
return_value=0.0,
|
||||
),
|
||||
):
|
||||
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
|
||||
assert results == [], f"drifted DOM should parse to no results, got {results!r}"
|
||||
@@ -274,6 +274,75 @@ def test_run_child_process_writes_failure_for_unexpected_exception(monkeypatch,
|
||||
assert "plain SeleniumBase startup failure" in result["traceback"]
|
||||
|
||||
|
||||
def test_run_child_process_applies_parent_dns_config(monkeypatch, tmp_path):
|
||||
"""Regression test for issue #1028: the helper subprocess must mirror the parent's
|
||||
DNS provider, otherwise it pre-resolves AA hostnames against (possibly hijacked)
|
||||
system DNS and Chrome loads the wrong page."""
|
||||
import io
|
||||
import json
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
result_path = tmp_path / "result.json"
|
||||
request = {
|
||||
"url": "https://annas-archive.pk/slow_download/abc/0/0",
|
||||
"retry": 1,
|
||||
"result_path": str(result_path),
|
||||
"dns_config": {
|
||||
"provider": "cloudflare",
|
||||
"servers": ["1.1.1.1", "1.0.0.1"],
|
||||
"doh_url": "https://cloudflare-dns.com/dns-query",
|
||||
"doh_enabled": True,
|
||||
"is_auto_mode": True,
|
||||
},
|
||||
}
|
||||
|
||||
applied: list[tuple] = []
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.network,
|
||||
"set_dns_provider",
|
||||
lambda provider, manual=None, *, use_doh=None: applied.append((provider, manual, use_doh)),
|
||||
)
|
||||
monkeypatch.setattr(internal_bypasser, "get", lambda *_a, **_k: "<html>ok</html>")
|
||||
monkeypatch.setattr(internal_bypasser.sys, "stdin", io.StringIO(json.dumps(request)))
|
||||
|
||||
assert internal_bypasser._run_child_process() == 0
|
||||
assert applied == [("cloudflare", None, True)]
|
||||
|
||||
|
||||
def test_apply_parent_dns_config_skips_auto_and_empty(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
calls: list = []
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.network,
|
||||
"set_dns_provider",
|
||||
lambda *a, **k: calls.append((a, k)),
|
||||
)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config({"provider": "auto"})
|
||||
internal_bypasser._apply_parent_dns_config({})
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_apply_parent_dns_config_forwards_manual_servers(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
calls: list = []
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.network,
|
||||
"set_dns_provider",
|
||||
lambda provider, manual=None, *, use_doh=None: calls.append((provider, manual, use_doh)),
|
||||
)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "manual", "servers": ["9.9.9.9"], "doh_enabled": False}
|
||||
)
|
||||
|
||||
assert calls == [("manual", ["9.9.9.9"], False)]
|
||||
|
||||
|
||||
def test_prepare_child_browser_env_uses_writable_runtime_paths(monkeypatch, tmp_path):
|
||||
import stat
|
||||
|
||||
|
||||
@@ -368,6 +368,20 @@ def test_download_source_settings_include_direct_download_toggle():
|
||||
assert "Add your own mirror URLs" in toggle_field.description
|
||||
|
||||
|
||||
def test_download_source_settings_include_distant_path_language_toggle():
|
||||
from shelfmark.config.settings import download_source_settings
|
||||
|
||||
fields = download_source_settings()
|
||||
toggle_field = next(
|
||||
field
|
||||
for field in fields
|
||||
if getattr(field, "key", None) == "DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH"
|
||||
)
|
||||
|
||||
assert toggle_field.default is False
|
||||
assert "distant path" in toggle_field.description.lower()
|
||||
|
||||
|
||||
def test_fast_source_options_lock_entries_without_mirror_or_donator_requirements(monkeypatch):
|
||||
from shelfmark.config.settings import _get_fast_source_options
|
||||
|
||||
|
||||
@@ -257,3 +257,119 @@ def test_entrypoint_root_bootstrap_fails_closed_when_config_repair_fails(tmp_pat
|
||||
assert f"Configured runtime identity: {os.getuid()}:{os.getgid()}" in result.stdout
|
||||
assert f"chown -R {os.getuid()}:{os.getgid()} /path/to/config" in result.stdout
|
||||
assert "Startup mode: root" not in result.stdout
|
||||
|
||||
|
||||
def test_entrypoint_rejects_wireguard_in_non_root_mode(tmp_path):
|
||||
result, _, _, _ = _run_entrypoint(tmp_path, extra_env={"USING_WIREGUARD": "true"})
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "USING_WIREGUARD=true requires the container to start as root." in result.stderr
|
||||
assert "Non-root mode skips the privileged network setup WireGuard depends on." in result.stderr
|
||||
|
||||
|
||||
def test_entrypoint_rejects_tor_and_wireguard_together(tmp_path):
|
||||
result, _, _, _ = _run_entrypoint(
|
||||
tmp_path,
|
||||
extra_env={"USING_TOR": "true", "USING_WIREGUARD": "true"},
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert (
|
||||
"USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode."
|
||||
in result.stderr
|
||||
)
|
||||
# The mutual-exclusion check must fire before either egress script runs, so
|
||||
# neither the Tor nor the WireGuard privileged-setup errors should appear.
|
||||
assert "requires the container to start as root" not in result.stderr
|
||||
|
||||
|
||||
def test_entrypoint_mutual_exclusion_precedes_tor_startup(tmp_path):
|
||||
# Even in root mode, enabling both must fail fast on mutual exclusion rather
|
||||
# than starting tor.sh and then aborting.
|
||||
result, _, _, _ = _run_entrypoint(
|
||||
tmp_path,
|
||||
simulate_root_startup=True,
|
||||
extra_env={"USING_TOR": "true", "USING_WIREGUARD": "true"},
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert (
|
||||
"USING_TOR and USING_WIREGUARD are mutually exclusive; enable only one egress mode."
|
||||
in result.stderr
|
||||
)
|
||||
|
||||
|
||||
def test_entrypoint_aborts_before_gunicorn_when_wireguard_fails(tmp_path):
|
||||
"""Security invariant: if wireguard.sh exits non-zero (any fail-closed path),
|
||||
entrypoint.sh must abort under `set -e` so gunicorn NEVER starts. A booting
|
||||
app after a failed egress setup would be a kill-switch bypass / IP leak.
|
||||
"""
|
||||
# Run the REAL entrypoint from a temp cwd that provides a stub `./wireguard.sh`
|
||||
# which exits 1, plus a stub `./tor.sh` (unused here) for completeness.
|
||||
work = tmp_path / "work"
|
||||
work.mkdir()
|
||||
real_entrypoint = ENTRYPOINT_PATH.read_text()
|
||||
(work / "entrypoint.sh").write_text(real_entrypoint)
|
||||
(work / "entrypoint.sh").chmod(0o755)
|
||||
_write_executable(
|
||||
work / "wireguard.sh",
|
||||
"#!/bin/sh\necho 'stub wireguard.sh failing closed' >&2\nexit 1\n",
|
||||
)
|
||||
_write_executable(work / "tor.sh", "#!/bin/sh\nexit 0\n")
|
||||
|
||||
bin_dir, runtime_home_file, runtime_args_file = _build_stub_bin(tmp_path)
|
||||
config_dir = tmp_path / "config"
|
||||
config_dir.mkdir(exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"BUILD_VERSION": "test-build",
|
||||
"CONFIG_DIR": str(config_dir),
|
||||
"DEBUG": "false",
|
||||
"ENABLE_LOGGING": "false",
|
||||
"ENTRYPOINT_GUNICORN_ARGS_FILE": str(runtime_args_file),
|
||||
"ENTRYPOINT_GUNICORN_HOME_FILE": str(runtime_home_file),
|
||||
"ENTRYPOINT_REAL_ID": ID_PATH,
|
||||
"ENTRYPOINT_REAL_MKDIR": MKDIR_PATH,
|
||||
"ENTRYPOINT_REAL_STAT": STAT_PATH,
|
||||
"ENTRYPOINT_STUB_GID": str(os.getgid()),
|
||||
"ENTRYPOINT_STUB_HOME": str(tmp_path / "runtime-home"),
|
||||
"ENTRYPOINT_STUB_UID": str(os.getuid()),
|
||||
# Root startup so the WireGuard branch runs ./wireguard.sh (our stub).
|
||||
"ENTRYPOINT_STUB_CURRENT_GID": "0",
|
||||
"ENTRYPOINT_STUB_CURRENT_UID": "0",
|
||||
"ENTRYPOINT_STUB_STAT_OWNER": "0:0",
|
||||
"FLASK_PORT": "8084",
|
||||
"LOG_LEVEL": "info",
|
||||
"LOG_ROOT": str(tmp_path / "logs"),
|
||||
"PATH": f"{bin_dir}:{env.get('PATH', '')}",
|
||||
"PGID": str(os.getgid()),
|
||||
"PUID": str(os.getuid()),
|
||||
"RELEASE_VERSION": "test-release",
|
||||
"TMP_DIR": str(tmp_path / "tmp"),
|
||||
"TZ": "",
|
||||
"USING_EXTERNAL_BYPASSER": "true",
|
||||
"USING_WIREGUARD": "true",
|
||||
}
|
||||
)
|
||||
|
||||
with _entrypoint_lock():
|
||||
result = subprocess.run(
|
||||
[BASH_PATH, str(work / "entrypoint.sh")],
|
||||
capture_output=True,
|
||||
cwd=work,
|
||||
env=env,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
# Entrypoint must have aborted with the stub's non-zero status...
|
||||
assert result.returncode != 0
|
||||
# ...the failure must actually come from wireguard.sh (not some unrelated
|
||||
# earlier abort), proven by the stub's marker on stderr...
|
||||
assert "stub wireguard.sh failing closed" in result.stderr
|
||||
# ...and gunicorn must NEVER have been invoked (args file never written).
|
||||
assert not runtime_args_file.exists(), (
|
||||
"gunicorn was started despite wireguard.sh failing — kill-switch bypass!"
|
||||
)
|
||||
|
||||
@@ -507,6 +507,15 @@ class TestConcurrencyConfiguration:
|
||||
assert interval >= 1
|
||||
assert interval <= 10
|
||||
|
||||
def test_completed_path_timeout_default(self):
|
||||
"""Completed external-client path wait should default to the legacy grace period."""
|
||||
from shelfmark.core.config import config
|
||||
|
||||
config.refresh()
|
||||
|
||||
timeout = config.get("DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT", 60)
|
||||
assert timeout == 60
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Cache Configuration Tests
|
||||
|
||||
@@ -1267,6 +1267,7 @@ class TestAdminSearchPreferences:
|
||||
assert data["keys"] == [
|
||||
"SEARCH_MODE",
|
||||
"SHOW_COMBINED_SELECTOR",
|
||||
"FORCE_COMBINED_SEARCH",
|
||||
"METADATA_PROVIDER",
|
||||
"METADATA_PROVIDER_AUDIOBOOK",
|
||||
"METADATA_PROVIDER_COMBINED",
|
||||
|
||||
@@ -597,3 +597,72 @@ def test_resolve_user_routes_expands_multiselect_event_rows(monkeypatch):
|
||||
{"event": "request_fulfilled", "url": "ntfys://ntfy.sh/user-main"},
|
||||
{"event": "all", "url": "ntfys://ntfy.sh/user-all"},
|
||||
]
|
||||
|
||||
|
||||
class TestAppriseProxyEnv:
|
||||
"""Regression tests for issue #956 — proxy settings ignored for notifications."""
|
||||
|
||||
def _patch_config(self, monkeypatch, values):
|
||||
from shelfmark.core import config as config_module
|
||||
|
||||
def _fake_get(key, default="", **_kwargs):
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(config_module.config, "get", _fake_get)
|
||||
|
||||
def test_http_proxy_mode_injects_proxy_env(self, monkeypatch):
|
||||
self._patch_config(
|
||||
monkeypatch,
|
||||
{
|
||||
"PROXY_MODE": "http",
|
||||
"HTTP_PROXY": "http://proxy.example.com:8080",
|
||||
"HTTPS_PROXY": "",
|
||||
"NO_PROXY": "",
|
||||
},
|
||||
)
|
||||
monkeypatch.delenv("HTTP_PROXY", raising=False)
|
||||
monkeypatch.delenv("HTTPS_PROXY", raising=False)
|
||||
|
||||
result = notifications_module._apprise_proxy_env()
|
||||
|
||||
assert result["HTTP_PROXY"] == "http://proxy.example.com:8080"
|
||||
assert result["HTTPS_PROXY"] == "http://proxy.example.com:8080"
|
||||
|
||||
def test_socks5_proxy_mode_injects_socks_env(self, monkeypatch):
|
||||
self._patch_config(
|
||||
monkeypatch,
|
||||
{
|
||||
"PROXY_MODE": "socks5",
|
||||
"SOCKS5_PROXY": "socks5://proxy.example.com:1080",
|
||||
"NO_PROXY": "",
|
||||
},
|
||||
)
|
||||
monkeypatch.delenv("HTTP_PROXY", raising=False)
|
||||
monkeypatch.delenv("HTTPS_PROXY", raising=False)
|
||||
|
||||
result = notifications_module._apprise_proxy_env()
|
||||
|
||||
assert result["HTTP_PROXY"] == "socks5://proxy.example.com:1080"
|
||||
assert result["HTTPS_PROXY"] == "socks5://proxy.example.com:1080"
|
||||
|
||||
def test_no_proxy_mode_returns_empty_dict(self, monkeypatch):
|
||||
self._patch_config(monkeypatch, {"PROXY_MODE": ""})
|
||||
|
||||
result = notifications_module._apprise_proxy_env()
|
||||
|
||||
assert result == {}
|
||||
|
||||
def test_does_not_override_already_set_env_vars(self, monkeypatch):
|
||||
self._patch_config(
|
||||
monkeypatch,
|
||||
{
|
||||
"PROXY_MODE": "http",
|
||||
"HTTP_PROXY": "http://new-proxy.example.com:8080",
|
||||
"NO_PROXY": "",
|
||||
},
|
||||
)
|
||||
monkeypatch.setenv("HTTP_PROXY", "http://existing-proxy.example.com:3128")
|
||||
|
||||
result = notifications_module._apprise_proxy_env()
|
||||
|
||||
assert "HTTP_PROXY" not in result
|
||||
|
||||
@@ -5,6 +5,31 @@ import types
|
||||
import xmlrpc.client as stdlib_xmlrpc_client
|
||||
|
||||
from shelfmark.core import utils
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
|
||||
class TestNormalizeHttpUrlQueryStripping:
|
||||
"""Regression tests for issue #999 — mirror URLs with query params/fragments."""
|
||||
|
||||
def test_strips_query_string_from_configured_url(self) -> None:
|
||||
result = normalize_http_url("http://mirror.example.com/search?token=abc123")
|
||||
assert result == "http://mirror.example.com/search"
|
||||
|
||||
def test_strips_fragment_from_configured_url(self) -> None:
|
||||
result = normalize_http_url("http://mirror.example.com/search#section")
|
||||
assert result == "http://mirror.example.com/search"
|
||||
|
||||
def test_strips_both_query_and_fragment(self) -> None:
|
||||
result = normalize_http_url("https://mirror.example.com/path?key=val&x=1#top")
|
||||
assert result == "https://mirror.example.com/path"
|
||||
|
||||
def test_plain_url_unchanged(self) -> None:
|
||||
result = normalize_http_url("http://mirror.example.com/search")
|
||||
assert result == "http://mirror.example.com/search"
|
||||
|
||||
def test_trailing_slash_still_stripped_after_query_removal(self) -> None:
|
||||
result = normalize_http_url("http://mirror.example.com/?token=x")
|
||||
assert result == "http://mirror.example.com"
|
||||
|
||||
|
||||
def test_get_hardened_xmlrpc_client_tolerates_patch_runtime_error(monkeypatch) -> None:
|
||||
|
||||
@@ -182,3 +182,170 @@ class TestDirectDownloadSearchQueries:
|
||||
("mistborn custom query", ["en"], ["epub"]),
|
||||
("mistborn custom query", None, ["epub"]),
|
||||
]
|
||||
|
||||
|
||||
# --- Distant-path language detection tests ---
|
||||
|
||||
|
||||
def _patch_path_language(monkeypatch, enabled: bool = True):
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
original_get = dd.config.get
|
||||
|
||||
def _fake_get(key: str, default=None, user_id=None):
|
||||
del user_id
|
||||
if key == "DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH":
|
||||
return enabled
|
||||
return original_get(key, default)
|
||||
|
||||
monkeypatch.setattr(dd.config, "get", _fake_get)
|
||||
return dd
|
||||
|
||||
|
||||
def _row_from_html(html: str):
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
return BeautifulSoup(html, "html.parser").find("tr")
|
||||
|
||||
|
||||
def _make_row(distant_path: str, language: str = "", record_id: str = "rec-1") -> str:
|
||||
return rf"""
|
||||
<tr>
|
||||
<td><a href="/md5/{record_id}"><img src="cover.jpg"></a></td>
|
||||
<td><span>A Book Title</span></td>
|
||||
<td><span>Author Name</span></td>
|
||||
<td><span>Publisher</span></td>
|
||||
<td><span>2024</span></td>
|
||||
<td><span>-</span></td>
|
||||
<td><span>-</span></td>
|
||||
<td><span>{language}</span></td>
|
||||
<td><span>fiction</span></td>
|
||||
<td><span>epub</span></td>
|
||||
<td><span>1 mb</span></td>
|
||||
<td><span>{distant_path}</span></td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
|
||||
def test_detects_bracketed_language_from_distant_path(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
row = _row_from_html(_make_row(r"lgli/N:\comics1\emule\2021.08.01\[BD FR] Scrameustache.cbz"))
|
||||
record = dd._parse_search_result_row(row)
|
||||
assert record is not None
|
||||
assert record.language == "fr"
|
||||
assert record.download_path is not None
|
||||
|
||||
|
||||
def test_detects_mixed_case_bracketed_language(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
row = _row_from_html(_make_row(r"lgli/V:\comics\_0DAY3\[Fr]\BDs [Fr]\!Pdf\S\Book.pdf"))
|
||||
record = dd._parse_search_result_row(row)
|
||||
assert record is not None
|
||||
assert record.language == "fr"
|
||||
|
||||
|
||||
def test_overrides_unknown_language_with_path_detection(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
row = _row_from_html(_make_row(r"lgli/V:\comics\_0DAY3\[Fr]\Book.pdf", language="unknown"))
|
||||
record = dd._parse_search_result_row(row)
|
||||
assert record is not None
|
||||
assert record.language == "fr"
|
||||
|
||||
|
||||
def test_sets_unknown_when_path_has_no_language(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
row = _row_from_html(_make_row(r"lgli/N:\comics1\emule\NoLanguageHere.epub"))
|
||||
record = dd._parse_search_result_row(row)
|
||||
assert record is not None
|
||||
assert record.language == "unknown"
|
||||
|
||||
|
||||
def test_avoids_en_false_positive_when_french_present(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
row = _row_from_html(
|
||||
_make_row(r"lgli/V:\comics\_0DAY2\Stripboeken Frans - BD en Français\[BD Fr] Book.cbr")
|
||||
)
|
||||
record = dd._parse_search_result_row(row)
|
||||
assert record is not None
|
||||
assert record.language == "fr"
|
||||
|
||||
|
||||
def test_keeps_row_with_missing_language_when_toggle_disabled(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch, enabled=False)
|
||||
row = _row_from_html(_make_row(r"lgli/N:\comics1\[BD FR] Scrameustache.cbz"))
|
||||
record = dd._parse_search_result_row(row)
|
||||
assert record is not None
|
||||
assert record.language is None
|
||||
|
||||
|
||||
def test_keeps_sparse_lgli_row(monkeypatch):
|
||||
"""lgli rows missing author/publisher/year must not be dropped."""
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
html = r"""
|
||||
<tr>
|
||||
<td><a href="/md5/sparse-1"><img src="cover.jpg"></a></td>
|
||||
<td><span>Gos - 1978 - Le scrameustache T06.cbz</span></td>
|
||||
<td></td><td></td><td></td><td></td><td></td><td></td>
|
||||
<td><span>Comic book</span></td>
|
||||
<td><span>cbz</span></td>
|
||||
<td><span>17.4MB</span></td>
|
||||
<td><span>lgli/N:\comics1\ftp\[BD.FR] French Comics\Book.cbz</span></td>
|
||||
</tr>
|
||||
"""
|
||||
record = dd._parse_search_result_row(_row_from_html(html))
|
||||
assert record is not None
|
||||
assert record.id == "sparse-1"
|
||||
assert record.language == "fr"
|
||||
assert record.author is None
|
||||
|
||||
|
||||
def test_search_books_filters_locally_when_path_language_enabled(monkeypatch):
|
||||
dd = _patch_path_language(monkeypatch)
|
||||
monkeypatch.setattr(dd.network, "get_aa_base_url", lambda: "https://mirror.example")
|
||||
monkeypatch.setattr(dd.network, "AAMirrorSelector", lambda: object())
|
||||
|
||||
captured_url: dict[str, str] = {}
|
||||
|
||||
def _fake_html_get_page(url: str, selector, allow_bypasser_fallback=False):
|
||||
del selector, allow_bypasser_fallback
|
||||
captured_url["url"] = url
|
||||
return r"""
|
||||
<table>
|
||||
<tr>
|
||||
<td><a href="/md5/rec-fr"><img src="c.jpg"></a></td>
|
||||
<td><span>Livre FR</span></td><td><span>Auteur</span></td>
|
||||
<td><span>Editeur</span></td><td><span>2025</span></td>
|
||||
<td><span>-</span></td><td><span>-</span></td><td></td>
|
||||
<td><span>fiction</span></td><td><span>pdf</span></td>
|
||||
<td><span>2 mb</span></td>
|
||||
<td><span>lgli/V:\comics\_0DAY3\[Fr]\Book FR.pdf</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="/md5/rec-en"><img src="c.jpg"></a></td>
|
||||
<td><span>Book EN</span></td><td><span>Author</span></td>
|
||||
<td><span>Publisher</span></td><td><span>2025</span></td>
|
||||
<td><span>-</span></td><td><span>-</span></td><td></td>
|
||||
<td><span>fiction</span></td><td><span>pdf</span></td>
|
||||
<td><span>2 mb</span></td>
|
||||
<td><span>lgli/V:\comics\_0DAY3\[En]\Book EN.pdf</span></td>
|
||||
</tr>
|
||||
</table>
|
||||
"""
|
||||
|
||||
monkeypatch.setattr(dd.downloader, "html_get_page", _fake_html_get_page)
|
||||
|
||||
records = dd.search_books("demo", SearchFilters(lang=["fr"], format=["pdf"]))
|
||||
|
||||
assert "&lang=" not in captured_url["url"]
|
||||
assert len(records) == 1
|
||||
assert records[0].id == "rec-fr"
|
||||
assert records[0].language == "fr"
|
||||
|
||||
|
||||
def test_book_matches_requested_languages_logic():
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
assert dd._book_matches_requested_languages(None, {"fr"}) is True
|
||||
assert dd._book_matches_requested_languages(None, set()) is True
|
||||
assert dd._book_matches_requested_languages("en", {"fr"}) is False
|
||||
assert dd._book_matches_requested_languages("fr", {"fr"}) is True
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""DoH resolver integration against the e2e platform's mock DoH responder.
|
||||
|
||||
The config-cluster analysis flagged DNS/DoH as a recurring break surface (#1028,
|
||||
#108). A fully hermetic DoH-over-the-network profile isn't feasible in the HTTP
|
||||
docker platform (DoH provider URLs are HTTPS + IP-pinned), so we exercise the
|
||||
*real* ``DoHResolver`` client against the platform's mock ``doh`` role here, over
|
||||
plain HTTP on localhost. This runs in normal CI (not just the nightly docker
|
||||
matrix) and guards the DoH JSON-parsing path the app relies on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from wsgiref.simple_server import WSGIServer, make_server
|
||||
|
||||
import pytest
|
||||
|
||||
MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py"
|
||||
|
||||
|
||||
def _load_mock_doh_app(doh_map: str):
|
||||
"""Import the platform mock_services module wired for the ``doh`` role.
|
||||
|
||||
The module wires its routes at import time from ``MOCK_ROLE``/``DOH_MAP``, so
|
||||
those must be set before loading it.
|
||||
"""
|
||||
os.environ["MOCK_ROLE"] = "doh"
|
||||
os.environ["DOH_MAP"] = doh_map
|
||||
spec = importlib.util.spec_from_file_location("mock_doh_services", MOCK_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.app
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def doh_url():
|
||||
if not MOCK_PATH.exists():
|
||||
pytest.skip(f"platform mock not found at {MOCK_PATH}")
|
||||
app = _load_mock_doh_app("aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11")
|
||||
server: WSGIServer = make_server("127.0.0.1", 0, app)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
# Google JSON DoH style uses the /resolve endpoint.
|
||||
yield f"http://127.0.0.1:{server.server_port}/resolve"
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def _resolver(doh_url: str):
|
||||
from shelfmark.download.network import DoHResolver
|
||||
|
||||
# hostname/ip args are the DoH server's own identity (used only for recursion
|
||||
# avoidance); the localhost values here are irrelevant to the lookups under test.
|
||||
return DoHResolver(doh_url, "127.0.0.1", "127.0.0.1")
|
||||
|
||||
|
||||
def test_doh_resolves_mapped_host(doh_url) -> None:
|
||||
"""The real DoH client parses the mock's JSON answer into an A record."""
|
||||
assert _resolver(doh_url).resolve("aa.mock.test", "A") == ["172.30.0.10"]
|
||||
|
||||
|
||||
def test_doh_nxdomain_returns_empty_not_error(doh_url) -> None:
|
||||
"""An unmapped name yields an empty list (Status 3), not an exception —
|
||||
the path that, when mishandled, surfaced as silent download failures."""
|
||||
assert _resolver(doh_url).resolve("unmapped.invalid", "A") == []
|
||||
|
||||
|
||||
def test_doh_resolver_caches_within_ttl(doh_url) -> None:
|
||||
"""A second lookup is served from cache (the resolver's documented behaviour)."""
|
||||
resolver = _resolver(doh_url)
|
||||
first = resolver.resolve("cf.mock.test", "A")
|
||||
assert first == ["172.30.0.11"]
|
||||
assert ("cf.mock.test", "A") in resolver._cache
|
||||
assert resolver.resolve("cf.mock.test", "A") == first
|
||||
@@ -137,3 +137,135 @@ def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch):
|
||||
assert result == [
|
||||
(network.socket.AF_INET, network.socket.SOCK_STREAM, 6, "", ("203.0.113.10", 443))
|
||||
]
|
||||
|
||||
|
||||
def _addrinfo(ip):
|
||||
return [(2, 1, 6, "", (ip, 443))]
|
||||
|
||||
|
||||
class _FakeDoHResolver:
|
||||
def __init__(self, ips):
|
||||
self._ips = ips
|
||||
|
||||
def resolve(self, _hostname, _record_type):
|
||||
return list(self._ips)
|
||||
|
||||
|
||||
def test_build_detection_doh_resolver_uses_selected_provider(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(
|
||||
network,
|
||||
"DNS_PROVIDERS",
|
||||
[
|
||||
("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
|
||||
("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(network, "_current_dns_index", 1) # user selected quad9
|
||||
|
||||
resolver = network._build_detection_doh_resolver()
|
||||
|
||||
assert resolver is not None
|
||||
assert resolver.base_url == "https://dns.quad9.net/dns-query"
|
||||
assert resolver.hostname == "dns.quad9.net"
|
||||
assert resolver.ip == "9.9.9.9"
|
||||
|
||||
|
||||
def test_build_detection_doh_resolver_falls_back_to_first_provider(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(
|
||||
network,
|
||||
"DNS_PROVIDERS",
|
||||
[
|
||||
("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
|
||||
("quad9", ["9.9.9.9"], "https://dns.quad9.net/dns-query"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(network, "_current_dns_index", -1) # system / not yet rotated
|
||||
|
||||
resolver = network._build_detection_doh_resolver()
|
||||
|
||||
assert resolver is not None
|
||||
assert resolver.base_url == "https://cloudflare-dns.com/dns-query"
|
||||
assert resolver.ip == "1.1.1.1"
|
||||
|
||||
|
||||
def test_detect_dns_interference_flags_divergent_resolvers(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
# System DNS (hijacked) returns an ISP block-page IP; DoH returns the real one.
|
||||
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
|
||||
monkeypatch.setattr(
|
||||
network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
|
||||
)
|
||||
|
||||
result = network.detect_dns_interference("annas-archive.pk")
|
||||
|
||||
assert result == {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
|
||||
|
||||
|
||||
def test_detect_dns_interference_none_when_resolvers_agree(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("203.0.113.7"))
|
||||
monkeypatch.setattr(
|
||||
network, "_build_detection_doh_resolver", lambda: _FakeDoHResolver(["203.0.113.7"])
|
||||
)
|
||||
|
||||
assert network.detect_dns_interference("annas-archive.pk") is None
|
||||
|
||||
|
||||
def test_detect_dns_interference_none_when_doh_unavailable(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "original_getaddrinfo", lambda *a, **k: _addrinfo("198.51.100.1"))
|
||||
monkeypatch.setattr(network, "_build_detection_doh_resolver", lambda: None)
|
||||
|
||||
assert network.detect_dns_interference("annas-archive.pk") is None
|
||||
|
||||
|
||||
def test_detect_dns_interference_skips_ip_and_local(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
def _should_not_run():
|
||||
raise AssertionError("resolver should not be built for IP/local hosts")
|
||||
|
||||
monkeypatch.setattr(network, "_build_detection_doh_resolver", _should_not_run)
|
||||
|
||||
assert network.detect_dns_interference("1.2.3.4") is None
|
||||
assert network.detect_dns_interference("localhost") is None
|
||||
|
||||
|
||||
def test_note_possible_dns_interference_warns_once_and_sets_flag(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "_dns_interference_warned", set())
|
||||
monkeypatch.setattr(network, "_dns_interference_active", False)
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
network,
|
||||
"detect_dns_interference",
|
||||
lambda host: (
|
||||
calls.append(host) or {"system_ips": ["198.51.100.1"], "doh_ips": ["203.0.113.7"]}
|
||||
),
|
||||
)
|
||||
|
||||
assert network.note_possible_dns_interference("annas-archive.pk") is True
|
||||
assert network.dns_interference_detected() is True
|
||||
# A repeat check for the same host must not re-run the costly detection.
|
||||
assert network.note_possible_dns_interference("annas-archive.pk") is True
|
||||
assert calls == ["annas-archive.pk"]
|
||||
|
||||
|
||||
def test_note_possible_dns_interference_no_detection_keeps_flag_false(monkeypatch):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "_dns_interference_warned", set())
|
||||
monkeypatch.setattr(network, "_dns_interference_active", False)
|
||||
monkeypatch.setattr(network, "detect_dns_interference", lambda _host: None)
|
||||
|
||||
assert network.note_possible_dns_interference("annas-archive.pk") is False
|
||||
assert network.dns_interference_detected() is False
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Validate the e2e platform's webseed .torrent generator.
|
||||
|
||||
The ``full`` e2e profile relies on a tracker-less webseed torrent so a real
|
||||
qBittorrent can complete a real download from the mock origin over HTTP. If the
|
||||
generator emits malformed bencode or mismatched piece hashes, qBittorrent would
|
||||
silently never complete — so we cross-check the generator against shelfmark's own
|
||||
``bencode_decode`` / ``extract_info_hash_from_torrent`` here, in normal CI.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
bencode_decode,
|
||||
extract_info_hash_from_torrent,
|
||||
)
|
||||
|
||||
GEN_PATH = (
|
||||
Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "make_webseed_torrent.py"
|
||||
)
|
||||
|
||||
|
||||
def _load_generator():
|
||||
if not GEN_PATH.exists():
|
||||
pytest.skip(f"generator not found at {GEN_PATH}")
|
||||
spec = importlib.util.spec_from_file_location("make_webseed_torrent", GEN_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
PAYLOAD = b"E2E webseed payload \x00\x01\x02 " * 2000 # ~50 KiB -> multiple pieces
|
||||
NAME = "sample-book.epub"
|
||||
WEBSEED = "http://mock-aa/payload/sample-book.epub"
|
||||
|
||||
|
||||
def test_generated_torrent_decodes_with_shelfmark_bencode() -> None:
|
||||
gen = _load_generator()
|
||||
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=16384)
|
||||
|
||||
decoded, _ = bencode_decode(raw)
|
||||
assert isinstance(decoded, dict)
|
||||
info = decoded[b"info"]
|
||||
assert info[b"name"] == NAME.encode()
|
||||
assert info[b"length"] == len(PAYLOAD)
|
||||
# url-list (webseed) must point at the file the mock serves.
|
||||
assert decoded[b"url-list"] == [WEBSEED.encode()]
|
||||
# No tracker — the whole point is HTTP-only completion.
|
||||
assert b"announce" not in decoded
|
||||
|
||||
|
||||
def test_piece_hashes_match_payload_bytes() -> None:
|
||||
gen = _load_generator()
|
||||
piece_len = 16384
|
||||
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=piece_len)
|
||||
decoded, _ = bencode_decode(raw)
|
||||
pieces = decoded[b"info"][b"pieces"]
|
||||
|
||||
expected = b"".join(
|
||||
hashlib.sha1(PAYLOAD[i : i + piece_len]).digest() for i in range(0, len(PAYLOAD), piece_len)
|
||||
)
|
||||
assert pieces == expected, "piece hashes do not match payload — qbit would never complete"
|
||||
assert len(pieces) % 20 == 0
|
||||
|
||||
|
||||
def test_info_hash_matches_shelfmark_extractor() -> None:
|
||||
"""Our infohash helper must agree with shelfmark's torrent parser."""
|
||||
gen = _load_generator()
|
||||
raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
|
||||
|
||||
ours = gen.info_hash(raw)
|
||||
theirs = extract_info_hash_from_torrent(raw)
|
||||
assert theirs is not None
|
||||
assert ours.lower() == theirs.lower(), (ours, theirs)
|
||||
|
||||
|
||||
def test_generator_is_deterministic() -> None:
|
||||
gen = _load_generator()
|
||||
a = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
|
||||
b = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
|
||||
assert a == b, "torrent generation must be byte-deterministic for stable infohash"
|
||||
@@ -0,0 +1,7 @@
|
||||
# Runtime state created by run-e2e.sh: per-profile shelfmark logs, the app's
|
||||
# /config (settings, users.db, secrets), downloaded books, and staging dirs.
|
||||
.state/
|
||||
|
||||
# Python bytecode from the mock services + suite.
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1,231 @@
|
||||
# Shelfmark E2E Docker Testing Platform
|
||||
|
||||
A hermetic, container-based end-to-end platform that boots the real Shelfmark app
|
||||
against **controllable** dependencies — a fake Anna's Archive, a Cloudflare gate, a
|
||||
mock FlareSolverr bypasser, a mock Prowlarr + a real qBittorrent, custom DNS
|
||||
servers, HTTP/SOCKS5 proxies, and a Tor profile — and runs a cluster test suite
|
||||
under each **config profile**.
|
||||
|
||||
It exists to make the recurring bug clusters from the issue/PR analysis impossible
|
||||
to reintroduce silently. The biggest one — Tor/Cloudflare/bypasser (37 issues /
|
||||
67 fix PRs) — had almost no automated coverage; this platform changes that.
|
||||
|
||||
```
|
||||
pytest suite (host :8084)
|
||||
│
|
||||
▼
|
||||
shelfmark (under test) ── egress depends on the active profile:
|
||||
├─ direct ──────────────► mock-aa (.10) fake Anna's Archive
|
||||
├─ Cloudflare gate ─────► mock-cf (.11) ─► mock-aa [full: real Chrome solves it]
|
||||
├─ FlareSolverr ────────► mock-cf (.11) ─► mock-flaresolverr (.12) [bypasser-external]
|
||||
├─ custom DNS ──────────► coredns (.20) / coredns-blocked (.22) [dns-manual/blocked]
|
||||
├─ HTTP / SOCKS proxy ──► tinyproxy (.30) / microsocks (.31) [proxy-http/socks]
|
||||
├─ Tor (transparent) ───► in-image tor.sh [tor]
|
||||
└─ Prowlarr → client ───► mock-prowlarr (.40) ─► qBittorrent [full: real download]
|
||||
|
||||
(all on one e2e docker network, 172.30.0.0/24, static IPs for DNS determinism)
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# one profile
|
||||
make e2e-platform # baseline
|
||||
make e2e-platform-profile PROFILE=bypasser-external
|
||||
make e2e-platform-profile PROFILE=client-deluge
|
||||
make e2e-platform-full # heavy: real Chrome + DoH + real qBittorrent
|
||||
|
||||
# the whole matrix
|
||||
make e2e-platform-matrix
|
||||
|
||||
# build the heavy image once, then reuse it (matrix does this automatically)
|
||||
make e2e-platform-build
|
||||
E2E_NO_BUILD=1 tests/e2e/platform/run-e2e.sh env/dns-doh.env
|
||||
|
||||
# debug: leave the stack up after the run
|
||||
KEEP_UP=1 tests/e2e/platform/run-e2e.sh env/dns-blocked.env
|
||||
```
|
||||
|
||||
Requirements: Docker + Compose v2, and `uv` (for the pytest runner). The runner
|
||||
builds the Shelfmark image from the repo `Dockerfile`, boots the profile's stack,
|
||||
waits for `/api/health`, runs the suite, and tears down. `run-matrix.sh` builds the
|
||||
image **once** and reuses it across profiles (`E2E_NO_BUILD=1`) so the slow
|
||||
xvfb/chromium layer isn't rebuilt per profile.
|
||||
|
||||
## How profiles work
|
||||
|
||||
Each profile is an env file in `env/`. It sets:
|
||||
- `COMPOSE_PROFILES` — which optional services start (compose `profiles:`).
|
||||
- `SM_*` — the app's config, injected as container env. Shelfmark treats
|
||||
deployment ENV as authoritative (`config.get`: "Deployment-level ENV values
|
||||
always win"), so a profile fully determines the app's DNS/proxy/bypasser/source
|
||||
configuration with no runtime mutation.
|
||||
- `E2E_PROFILE` — handed to pytest so the suite selects applicable tests.
|
||||
|
||||
Tests declare applicability with `@pytest.mark.profiles(...)`. **A test with no
|
||||
marker is a profile-agnostic invariant and runs under every profile** — that is
|
||||
how one cluster test ("source must be reachable") becomes the config matrix.
|
||||
|
||||
## The matrix (cluster × profile)
|
||||
|
||||
Status column: ✅ = run live on Docker and passing. Every profile below was run
|
||||
end-to-end (`docker compose up` + suite + teardown) and passes.
|
||||
|
||||
| Profile | Egress / what it proves | Clusters | Regression targets | Status |
|
||||
|---|---|---|---|---|
|
||||
| `baseline` | Direct to fake AA; search/parse + #1028 clean-failure | 2,3,4 | #198 #293 #214 #1040 #1028 | ✅ 9 passed |
|
||||
| `bypasser-external` | External bypasser wired; CF-gated search fails cleanly | 1 | #284 #202 #410 #369 | ✅ 5 passed |
|
||||
| `bypasser-disabled` | CF-gated AA + bypasser OFF → no results (control) | 1 | #202 #410 | ✅ 4 passed |
|
||||
| `dns-manual` | AA only resolvable via custom DNS (coredns) | config: DNS | #108 | ✅ 4 passed |
|
||||
| `dns-blocked` | System DNS NXDOMAINs AA; custom DNS resolves it | config: DNS | **#1028** | ✅ 4 passed |
|
||||
| `dns-doh` | System DNS blocks AA; **DoH over real HTTPS** resolves it | config: DoH | **#1028** #108 | ✅ 3 passed |
|
||||
| `proxy-http` | All egress via tinyproxy, **proven by proxy logs** | config: proxy | **#956** | ✅ 6 passed |
|
||||
| `proxy-socks` | All egress via SOCKS5 (microsocks), traversal-checked | config: proxy | #956 | ✅ 5 passed |
|
||||
| `tor` | `USING_TOR=true` boots clean (restarts=0) | 1/6 Tor boot | #1021 #940 #801 | ✅ 5 passed |
|
||||
| `client-transmission` | Prowlarr → **real Transmission** webseed download → /books | 5 clients | #1022 #634 | ✅ 4 passed |
|
||||
| `client-deluge` | Prowlarr → **real Deluge** webseed download → /books | 5 clients | #530 | ✅ 4 passed |
|
||||
| `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed |
|
||||
| *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ |
|
||||
|
||||
> **The bypasser is download-time, not search-time.** Running the stack revealed
|
||||
> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a
|
||||
> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser
|
||||
> (internal Chrome or external FlareSolverr) only runs during a file *download*
|
||||
> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean*
|
||||
> CF-gated-search failure, while the **`full` profile exercises the real end-to-end
|
||||
> CF solve**: AA search/detail are reachable, but the AA slow-download link points
|
||||
> at the gate, so downloading Moby-Dick forces the in-image headless Chromium to
|
||||
> detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file —
|
||||
> verified live (`Challenge detected: cloudflare` → `Bypass successful` → Moby-Dick
|
||||
> in `/books`).
|
||||
>
|
||||
> **`bypasser-external` must set `SM_USING_EXTERNAL_BYPASSER=true`** — shelfmark does
|
||||
> **not** derive it from `EXT_BYPASSER_URL`; without it the app silently uses the
|
||||
> in-image Chrome bypasser instead of FlareSolverr.
|
||||
|
||||
Coverage of the 7 clusters from the analysis:
|
||||
|
||||
1. **Bypasser/Tor/Cloudflare** → `bypasser-external`, `bypasser-disabled`, `tor`.
|
||||
2. **Search/metadata** → `baseline` (`test_cluster_search_aa.py`, hermetic via the
|
||||
`direct_download` source so no external metadata provider is needed).
|
||||
3. **AA parsing/mirrors** → `baseline` parse guards incl. the **layout-drift
|
||||
fail-loud** test (#878/#879/#880).
|
||||
4. **Permissions/file-move** → `baseline` (`test_cluster_download_permissions.py`).
|
||||
5. **Torrent/usenet clients** → a mock Prowlarr + webseed torrent drives **three
|
||||
real torrent clients** end to end (`full`=qBittorrent, `client-transmission`,
|
||||
`client-deluge`) — completion detection + file move into `/books`. One
|
||||
client-agnostic test (`test_cluster_clients.py`) covers all three.
|
||||
6. **Docker/entrypoint/PUID-PGID** → profile-agnostic health + boot-log checks,
|
||||
run under every profile.
|
||||
7. **Audiobook/ABB** → parse-contract guards in
|
||||
`tests/audiobookbay/test_scraper_contract.py` (info-hash normalization #386,
|
||||
magnet fallback, layout drift). These run in **normal CI**, not the docker
|
||||
matrix, because ABB hardcodes `https://` for its fetches (see Roadmap).
|
||||
|
||||
### Proxy traversal (not just reachability)
|
||||
Because the app and the mock AA share the e2e network, a regression that ignores
|
||||
the proxy config would still reach AA directly. `test_egress_actually_traverses_proxy`
|
||||
drives a search and then inspects the proxy container's logs, so the proxy
|
||||
profiles prove the egress *went through* the proxy — a real guard for #956.
|
||||
|
||||
### DoH — two layers
|
||||
- **Offline** (`tests/download/test_doh_resolver_mock.py`, normal CI): the real
|
||||
`DoHResolver` is driven against the mock `doh` role over localhost HTTP, covering
|
||||
JSON-answer parsing, NXDOMAIN → empty, and caching.
|
||||
- **In-stack** (`dns-doh` profile): the mock `doh` role serves the DNS JSON API over
|
||||
**real HTTPS** (self-signed). The system resolver (coredns-blocked) NXDOMAINs
|
||||
`aa.mock.test`, so the host can *only* be resolved via DoH; compose `extra_hosts`
|
||||
redirects the `cloudflare-dns.com` provider to the in-stack mock and
|
||||
`CERTIFICATE_VALIDATION=disabled` accepts the self-signed cert. The search reaching
|
||||
AA proves the app's DoH path resolved the name end to end — **no app code change**.
|
||||
|
||||
### The `full` profile — real Chrome + real client (`make e2e-platform-full`)
|
||||
The "everything real" heavy profile (test book: **Moby-Dick**), run nightly / on
|
||||
demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, and
|
||||
**passes live** (6 passed):
|
||||
|
||||
- **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are
|
||||
reachable (`mock-aa`), but the AA *slow-download* link points at the Cloudflare
|
||||
gate (`mock-cf`), whose challenge page runs JS that issues `cf_clearance` and
|
||||
reloads. Downloading Moby-Dick forces the in-image headless Chromium (seleniumbase
|
||||
CDP, in the `shelfmark` image via `xvfb`+`chromium`) to load the gate, detect the
|
||||
challenge (`Challenge detected: cloudflare`), solve it (`_bypass_method_cdp_solve`),
|
||||
and fetch the cleared "Download now" page → the file lands in `/books`. That
|
||||
outcome is *only* reachable if Chrome solved the gate — the literal "spin a Chrome
|
||||
browser" path and the strongest guard for the #1 cluster. Two subtleties this
|
||||
surfaced, now handled by the mock: the cleared page must exceed the bypasser's
|
||||
`_LOADING_BODY_LENGTH_MAX` (50 chars of innerText) or it loops as "still loading",
|
||||
and the AA detail page must satisfy the brittle `original_nodes[-6]` parse (#880).
|
||||
- **DoH** on at boot.
|
||||
- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`,
|
||||
`/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a
|
||||
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real
|
||||
qBittorrent completes the download over HTTP (no tracker/peer/seeder), and
|
||||
shelfmark's completion detection + file move lands the book in `/books`. The
|
||||
webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked
|
||||
against shelfmark's own bencode/infohash parser in
|
||||
`tests/download/test_webseed_torrent_generator.py`), and the whole
|
||||
prowlarr→qBittorrent path is configured declaratively via env (`env/full.env`).
|
||||
|
||||
## Components
|
||||
|
||||
| Path | Purpose |
|
||||
|---|---|
|
||||
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. |
|
||||
| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. |
|
||||
| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. |
|
||||
| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. |
|
||||
| `mocks/fixtures/*.html` | AA search/detail HTML in the **exact** shape the parser expects, plus drift/empty/no-files variants. |
|
||||
| `docker-compose.e2e.yml` | The stack; optional services gated by compose profiles, static IPs for DNS determinism. |
|
||||
| `dns/Corefile*`, `dns/mock.test.db` | coredns zones — working + ISP-block (NXDOMAIN). |
|
||||
| `env/*.env` | The config profiles (matrix rows). |
|
||||
| `suite/` | The pytest harness + cluster tests. |
|
||||
| `run-e2e.sh` / `run-matrix.sh` | Boot one profile / loop the matrix. |
|
||||
| `build-images.sh` | Build the heavy image once (`make e2e-platform-build`); reused via `E2E_NO_BUILD=1`. |
|
||||
|
||||
### Fault injection
|
||||
The mock AA reproduces historical bugs deterministically. Injection rides inside
|
||||
the search query as `E2EINJECT:<name>` (the app builds the AA URL itself and only
|
||||
forwards the user query as `q=`). Names: `no_files`, `empty`, `layout_drift`,
|
||||
`500`. The harness embeds them via `PlatformClient.direct_search(..., inject=...)`.
|
||||
|
||||
## Gating PRs (block merge on e2e failure)
|
||||
|
||||
The `.github/workflows/e2e-platform.yml` workflow runs on every PR. On a PR that
|
||||
touches relevant code (`shelfmark/**`, `Dockerfile`, `entrypoint.sh`, `tor.sh`,
|
||||
`tests/e2e/platform/**`) it runs the fast PR subset **and** the heavy `full`
|
||||
profile (real Chrome solving Cloudflare + DoH + real qBittorrent), then a single
|
||||
**`e2e required`** job aggregates them: it fails if any e2e job failed, and passes
|
||||
(so it never hangs) when the e2e jobs are skipped on an unrelated PR.
|
||||
|
||||
The workflow producing a failing check is **not enough on its own** — GitHub only
|
||||
*blocks merge* on checks listed in branch protection. A repo **admin** must, once:
|
||||
|
||||
- **UI:** Settings → Branches → branch protection rule for `main` →
|
||||
*Require status checks to pass before merging* → add **`e2e required`**.
|
||||
- **or `gh` (admin token):**
|
||||
```bash
|
||||
gh api -X PUT repos/calibrain/shelfmark/branches/main/protection \
|
||||
-H "Accept: application/vnd.github+json" --input - <<'JSON'
|
||||
{ "required_status_checks": { "strict": true, "contexts": ["e2e required"] },
|
||||
"enforce_admins": true, "required_pull_request_reviews": null, "restrictions": null }
|
||||
JSON
|
||||
```
|
||||
|
||||
After that, any failure in the e2e platform tests (including the `full` profile)
|
||||
blocks the PR from merging. Requiring just the one `e2e required` context covers
|
||||
the whole dynamic matrix, so the list never needs updating as profiles change.
|
||||
|
||||
## Known limitations / follow-ups
|
||||
|
||||
- **rTorrent.** Not in the matrix: its rakshasa-libtorrent has **no GetRight/webseed
|
||||
support**, so the hermetic webseed torrent (which qBittorrent/Transmission/Deluge
|
||||
all complete) leaves rTorrent stuck at 0%. Supporting it needs a real tracker +
|
||||
seeder (peer download) — a follow-up that the webseed design intentionally avoids.
|
||||
- **Usenet clients (SABnzbd/NZBGet).** Not yet covered — completing a usenet download
|
||||
hermetically needs a mock NNTP server serving the yEnc-encoded payload plus an NZB,
|
||||
which is a separate (larger) build than the torrent webseed path.
|
||||
- **Audiobook (cluster 7) in-stack.** ABB hardcodes `https://`, so it's covered
|
||||
offline (`tests/audiobookbay/test_scraper_contract.py`); an in-stack
|
||||
`audiobookbay` role needs the same self-signed-HTTPS plumbing the `dns-doh` profile
|
||||
now uses for DoH.
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build every buildable image in the e2e stack once (the heavy `shelfmark` image
|
||||
# plus the mock-* role images), so run-matrix.sh / run-e2e.sh with E2E_NO_BUILD=1
|
||||
# can reuse them instead of rebuilding the xvfb/chromium layer per profile.
|
||||
set -euo pipefail
|
||||
|
||||
PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$PLATFORM_DIR"
|
||||
|
||||
# Activate every profile that owns a buildable service so they all get built.
|
||||
# (Download clients, coredns, proxies are pre-built images — nothing to build.)
|
||||
export COMPOSE_PROFILES="bypasser-external,full,dns-doh"
|
||||
echo "==> building shelfmark + mock images (one cold build of the chromium layer)"
|
||||
docker compose -f docker-compose.e2e.yml build
|
||||
echo "==> done. Reuse with: E2E_NO_BUILD=1 ./run-e2e.sh env/<profile>.env"
|
||||
@@ -0,0 +1,13 @@
|
||||
# coredns: authoritative for mock.test, forwards everything else.
|
||||
mock.test:53 {
|
||||
file /zones/mock.test.db
|
||||
log
|
||||
errors
|
||||
}
|
||||
|
||||
. :53 {
|
||||
forward . 1.1.1.1 8.8.8.8
|
||||
cache 30
|
||||
log
|
||||
errors
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# coredns: ISP-DNS-block simulation. Resolves nothing under mock.test
|
||||
# (returns NXDOMAIN), so an app that relies on *system* DNS for book sources
|
||||
# fails — while an app that uses its own custom DNS resolver still works.
|
||||
# Regression harness for #1028 (internal bypasser used system DNS in subprocess).
|
||||
mock.test:53 {
|
||||
template IN ANY mock.test {
|
||||
rcode NXDOMAIN
|
||||
}
|
||||
log
|
||||
errors
|
||||
}
|
||||
|
||||
. :53 {
|
||||
# Everything else still resolves, so only the book-source host is "blocked".
|
||||
forward . 1.1.1.1 8.8.8.8
|
||||
log
|
||||
errors
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
$ORIGIN mock.test.
|
||||
$TTL 60
|
||||
@ IN SOA ns.mock.test. admin.mock.test. (
|
||||
1 ; serial
|
||||
7200 ; refresh
|
||||
3600 ; retry
|
||||
1209600 ; expire
|
||||
3600 ) ; minimum
|
||||
@ IN NS ns.mock.test.
|
||||
ns IN A 172.30.0.20
|
||||
aa IN A 172.30.0.10
|
||||
cf IN A 172.30.0.11
|
||||
doh IN A 172.30.0.21
|
||||
@@ -0,0 +1,332 @@
|
||||
# Shelfmark e2e Docker testing platform
|
||||
# ---------------------------------------------------------------------------
|
||||
# A hermetic stack: the app under test plus *controllable* dependencies
|
||||
# (fake Anna's Archive, a Cloudflare gate, a mock FlareSolverr bypasser, a DoH
|
||||
# responder, a DNS server, HTTP + SOCKS5 proxies, and a Tor profile).
|
||||
#
|
||||
# Config profiles are env-driven: pick a profile with an env file under env/
|
||||
# (it sets COMPOSE_PROFILES + the SM_* config the app boots with). The runner
|
||||
# brings the stack up per profile, runs the matching cluster tests, tears down.
|
||||
#
|
||||
# ./run-e2e.sh env/baseline.env
|
||||
# ./run-e2e.sh env/bypasser-external.env
|
||||
# ./run-e2e.sh env/dns-blocked.env
|
||||
#
|
||||
# See README.md for the full cluster x profile matrix.
|
||||
|
||||
x-mock-build: &mock-build
|
||||
build:
|
||||
context: ./mocks
|
||||
dockerfile: Dockerfile
|
||||
|
||||
networks:
|
||||
e2e:
|
||||
driver: bridge
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.30.0.0/24
|
||||
|
||||
services:
|
||||
# ----- App under test --------------------------------------------------- #
|
||||
shelfmark:
|
||||
build:
|
||||
context: ../../..
|
||||
dockerfile: Dockerfile
|
||||
target: ${SM_BUILD_TARGET:-shelfmark}
|
||||
container_name: e2e-shelfmark
|
||||
cap_add:
|
||||
- NET_ADMIN # required by tor.sh iptables when USING_TOR=true
|
||||
- NET_RAW
|
||||
environment:
|
||||
TZ: UTC
|
||||
DEBUG: "true"
|
||||
ONBOARDING: "false" # skip wizard; ephemeral storage
|
||||
DISABLE_LOCAL_AUTH: "${SM_DISABLE_LOCAL_AUTH:-true}"
|
||||
PUID: "${SM_PUID:-1000}"
|
||||
PGID: "${SM_PGID:-1000}"
|
||||
# --- source / mirror config (cluster 2/3) ---
|
||||
DIRECT_DOWNLOAD_ENABLED: "${SM_DIRECT_DOWNLOAD_ENABLED:-true}"
|
||||
AA_ADDITIONAL_URLS: "${SM_AA_URL:-http://mock-aa}"
|
||||
# --- bypasser config (cluster 1) ---
|
||||
USE_CF_BYPASS: "${SM_USE_CF_BYPASS:-false}"
|
||||
# Selects the external (FlareSolverr) bypasser; when false the in-image
|
||||
# Chrome (internal) bypasser is used. NOT derived from EXT_BYPASSER_URL.
|
||||
USING_EXTERNAL_BYPASSER: "${SM_USING_EXTERNAL_BYPASSER:-false}"
|
||||
EXT_BYPASSER_URL: "${SM_EXT_BYPASSER_URL:-}"
|
||||
# --- prowlarr indexer + torrent client (cluster 5, `full` profile) ---
|
||||
PROWLARR_ENABLED: "${SM_PROWLARR_ENABLED:-false}"
|
||||
PROWLARR_URL: "${SM_PROWLARR_URL:-}"
|
||||
PROWLARR_API_KEY: "${SM_PROWLARR_API_KEY:-}"
|
||||
PROWLARR_TORRENT_CLIENT: "${SM_PROWLARR_TORRENT_CLIENT:-}"
|
||||
QBITTORRENT_URL: "${SM_QBITTORRENT_URL:-}"
|
||||
QBITTORRENT_USERNAME: "${SM_QBITTORRENT_USERNAME:-}"
|
||||
QBITTORRENT_PASSWORD: "${SM_QBITTORRENT_PASSWORD:-}"
|
||||
QBITTORRENT_CATEGORY: "${SM_QBITTORRENT_CATEGORY:-}"
|
||||
DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT: "${SM_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT:-60}"
|
||||
# transmission / deluge / rtorrent (client-* profiles)
|
||||
TRANSMISSION_URL: "${SM_TRANSMISSION_URL:-}"
|
||||
TRANSMISSION_USERNAME: "${SM_TRANSMISSION_USERNAME:-}"
|
||||
TRANSMISSION_PASSWORD: "${SM_TRANSMISSION_PASSWORD:-}"
|
||||
DELUGE_HOST: "${SM_DELUGE_HOST:-}"
|
||||
DELUGE_PORT: "${SM_DELUGE_PORT:-}"
|
||||
DELUGE_PASSWORD: "${SM_DELUGE_PASSWORD:-}"
|
||||
RTORRENT_URL: "${SM_RTORRENT_URL:-}"
|
||||
# --- DNS / DoH (config cluster) ---
|
||||
CUSTOM_DNS: "${SM_CUSTOM_DNS:-}"
|
||||
CUSTOM_DNS_MANUAL: "${SM_CUSTOM_DNS_MANUAL:-}"
|
||||
USE_DOH: "${SM_USE_DOH:-false}"
|
||||
# Disable TLS verification so the in-stack DoH-over-HTTPS mock (self-signed)
|
||||
# is accepted in the dns-doh profile. Default keeps verification ON.
|
||||
CERTIFICATE_VALIDATION: "${SM_CERTIFICATE_VALIDATION:-enabled}"
|
||||
# --- proxy (config cluster) ---
|
||||
PROXY_MODE: "${SM_PROXY_MODE:-none}"
|
||||
HTTP_PROXY_URL: "${SM_HTTP_PROXY:-}"
|
||||
HTTP_PROXY: "${SM_HTTP_PROXY:-}"
|
||||
SOCKS5_PROXY: "${SM_SOCKS5_PROXY:-}"
|
||||
NO_PROXY: "${SM_NO_PROXY:-}"
|
||||
# --- tor (cluster 1/6) ---
|
||||
USING_TOR: "${SM_USING_TOR:-false}"
|
||||
ports:
|
||||
- "8084:8084"
|
||||
volumes:
|
||||
- ./.state/config:/config
|
||||
- ./.state/books:/books
|
||||
- ${SM_DOWNLOADS_HOST_DIR:-./.state/downloads}:/downloads
|
||||
- ./.state/tmp:/tmp/shelfmark
|
||||
networks:
|
||||
- e2e
|
||||
dns:
|
||||
# When a DNS profile is active, point the container's system resolver at
|
||||
# our controllable server; otherwise Docker's embedded DNS (127.0.0.11).
|
||||
- ${SM_SYSTEM_DNS:-127.0.0.11}
|
||||
extra_hosts:
|
||||
# Redirect the DoH provider hostname to the in-stack mock-doh (dns-doh
|
||||
# profile). Harmless elsewhere — only the dns-doh profile enables DoH against
|
||||
# the cloudflare provider, and /etc/hosts is consulted before the resolver.
|
||||
- "cloudflare-dns.com:172.30.0.21"
|
||||
restart: "no"
|
||||
|
||||
# ----- Fake Anna's Archive origin (always on) --------------------------- #
|
||||
mock-aa:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-aa
|
||||
environment:
|
||||
MOCK_ROLE: origin-aa
|
||||
# When set (the `full` profile sets it to the CF gate), AA slow-download
|
||||
# links point through Cloudflare so a real download forces the internal
|
||||
# Chrome bypasser to solve the challenge. Empty -> same-origin (no CF).
|
||||
SLOW_DOWNLOAD_BASE: "${SM_SLOW_DOWNLOAD_BASE:-}"
|
||||
AA_FILE_BASE: "http://mock-aa"
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.10
|
||||
aliases:
|
||||
- aa.mock.test
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://localhost/healthz')"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
# ----- Cloudflare gate (profiles: bypasser-external, full) -------------- #
|
||||
# `full` puts AA's slow-download behind this gate so the real Chrome bypasser
|
||||
# must solve it; `bypasser-external` puts AA search behind it for the
|
||||
# FlareSolverr negative/positive controls.
|
||||
mock-cf:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-cf
|
||||
profiles: ["bypasser-external", "full"]
|
||||
environment:
|
||||
MOCK_ROLE: cloudflare
|
||||
ORIGIN_INTERNAL_URL: http://mock-aa
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.11
|
||||
aliases:
|
||||
- cf.mock.test
|
||||
|
||||
# ----- Mock FlareSolverr external bypasser (profile: bypasser-external) -- #
|
||||
mock-flaresolverr:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-flaresolverr
|
||||
profiles: ["bypasser-external"]
|
||||
environment:
|
||||
MOCK_ROLE: flaresolverr
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.12
|
||||
|
||||
# NOTE: the mock `doh` role lives in mock_services.py and is exercised by
|
||||
# tests/download/test_doh_resolver_mock.py (real DoHResolver over localhost HTTP).
|
||||
# An in-stack DoH service is intentionally absent — see README "Known limitations".
|
||||
|
||||
# ----- DNS server (profile: dns-manual) --------------------------------- #
|
||||
coredns:
|
||||
image: coredns/coredns:1.11.1
|
||||
container_name: e2e-coredns
|
||||
profiles: ["dns-manual"]
|
||||
command: ["-conf", "/Corefile"]
|
||||
volumes:
|
||||
- ./dns/Corefile:/Corefile:ro
|
||||
- ./dns/mock.test.db:/zones/mock.test.db:ro
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.20
|
||||
|
||||
# ----- DNS server that NXDOMAINs the AA host (profile: dns-blocked) ------ #
|
||||
# Simulates ISP DNS blocking (#1028); the app must fall back (DoH / direct).
|
||||
coredns-blocked:
|
||||
image: coredns/coredns:1.11.1
|
||||
container_name: e2e-coredns-blocked
|
||||
profiles: ["dns-blocked"]
|
||||
command: ["-conf", "/Corefile"]
|
||||
volumes:
|
||||
- ./dns/Corefile.blocked:/Corefile:ro
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.22
|
||||
|
||||
# ----- HTTP proxy (profile: proxy-http) --------------------------------- #
|
||||
tinyproxy:
|
||||
image: monokal/tinyproxy:latest
|
||||
container_name: e2e-tinyproxy
|
||||
profiles: ["proxy-http"]
|
||||
command: ANY
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.30
|
||||
|
||||
# ----- SOCKS5 proxy (profile: proxy-socks) ------------------------------ #
|
||||
microsocks:
|
||||
image: vimagick/microsocks:latest
|
||||
container_name: e2e-microsocks
|
||||
profiles: ["proxy-socks"]
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.31
|
||||
|
||||
# ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- #
|
||||
# Minimal Prowlarr API returning one torrent release whose .torrent is a
|
||||
# webseed pointing at mock-aa. Drives the real torrent-client download in the
|
||||
# `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles.
|
||||
mock-prowlarr:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-prowlarr
|
||||
profiles: ["full", "client-transmission", "client-deluge", "client-qbittorrent-delayed"]
|
||||
environment:
|
||||
MOCK_ROLE: prowlarr
|
||||
AA_INTERNAL_URL: http://mock-aa
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.40
|
||||
aliases:
|
||||
- prowlarr.mock.test
|
||||
|
||||
# ----- Real qBittorrent download client (profiles: full, delayed path) --- #
|
||||
# Auth is bypassed for the e2e subnet (qBittorrent.conf) so shelfmark connects
|
||||
# without juggling the image's random temp password.
|
||||
qbittorrent:
|
||||
image: lscr.io/linuxserver/qbittorrent:latest
|
||||
container_name: e2e-qbittorrent
|
||||
profiles: ["full", "client-qbittorrent-delayed"]
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: UTC
|
||||
WEBUI_PORT: "8080"
|
||||
volumes:
|
||||
- ./qbittorrent/qBittorrent.conf:/config/qBittorrent/qBittorrent.conf
|
||||
# Normally shared with Shelfmark. In the delayed profile this points at a
|
||||
# client-only directory and the sync sidecar makes it visible later.
|
||||
- ${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/downloads}:/downloads
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# ----- Delayed file visibility sidecar (profile: client-qbittorrent-delayed) #
|
||||
# Simulates a seedbox/remote-sync workflow: qBittorrent writes into one host
|
||||
# directory while Shelfmark sees a separate /downloads directory that is synced
|
||||
# only after a delay. This exercises DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT.
|
||||
delayed-download-sync:
|
||||
image: alpine:3.20
|
||||
container_name: e2e-delayed-download-sync
|
||||
profiles: ["client-qbittorrent-delayed"]
|
||||
environment:
|
||||
SYNC_DELAY_SECONDS: "${SM_DELAYED_DOWNLOAD_SYNC_SECONDS:-20}"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
mkdir -p /synced
|
||||
while true; do
|
||||
found=0
|
||||
for path in /remote/*; do
|
||||
[ -e "$${path}" ] || continue
|
||||
found=1
|
||||
before="$$(du -sk /remote 2>/dev/null | awk '{print $$1}')"
|
||||
sleep "$${SYNC_DELAY_SECONDS}"
|
||||
after="$$(du -sk /remote 2>/dev/null | awk '{print $$1}')"
|
||||
if [ -n "$${before}" ] && [ "$${before}" = "$${after}" ]; then
|
||||
cp -a /remote/. /synced/ 2>/dev/null || true
|
||||
fi
|
||||
break
|
||||
done
|
||||
[ "$${found}" = "1" ] || sleep 1
|
||||
done
|
||||
volumes:
|
||||
- ${SM_QBITTORRENT_DOWNLOADS_HOST_DIR:-./.state/downloads}:/remote:ro
|
||||
- ${SM_DOWNLOADS_HOST_DIR:-./.state/downloads}:/synced
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# ----- Real Transmission client (profile: client-transmission) ---------- #
|
||||
# Same webseed torrent + mock Prowlarr as `full`, different real client.
|
||||
transmission:
|
||||
image: lscr.io/linuxserver/transmission:latest
|
||||
container_name: e2e-transmission
|
||||
profiles: ["client-transmission"]
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: UTC
|
||||
USER: admin
|
||||
PASS: admin
|
||||
volumes:
|
||||
- ./.state/downloads:/downloads
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# ----- Real Deluge client (profile: client-deluge) ---------------------- #
|
||||
# shelfmark talks to deluge-web (default WebUI password "deluge"), which
|
||||
# connects to the bundled daemon — no daemon auth-file juggling.
|
||||
deluge:
|
||||
image: lscr.io/linuxserver/deluge:latest
|
||||
container_name: e2e-deluge
|
||||
profiles: ["client-deluge"]
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: UTC
|
||||
volumes:
|
||||
- ./.state/downloads:/downloads
|
||||
networks:
|
||||
- e2e
|
||||
|
||||
# NOTE: rTorrent is intentionally absent — its rakshasa-libtorrent has no
|
||||
# GetRight/webseed support, so the hermetic webseed torrent can't drive it.
|
||||
# Supporting it would need a real tracker + seeder (see README limitations).
|
||||
|
||||
# ----- DoH-over-HTTPS responder (profile: dns-doh) ---------------------- #
|
||||
# In-stack DoH: serves the DNS JSON API over HTTPS with a self-signed cert.
|
||||
# shelfmark reaches it because the profile (a) maps the DoH provider hostname
|
||||
# to this container via extra_hosts and (b) sets CERTIFICATE_VALIDATION=disabled.
|
||||
mock-doh:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-doh
|
||||
profiles: ["dns-doh"]
|
||||
environment:
|
||||
MOCK_ROLE: doh
|
||||
DOH_TLS: "1"
|
||||
DOH_MAP: "aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11"
|
||||
networks:
|
||||
e2e:
|
||||
ipv4_address: 172.30.0.21
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
# Profile: baseline
|
||||
# Direct connection to the fake Anna's Archive, no bypasser, no proxy, system DNS.
|
||||
# Exercises clusters 2/3 (search + AA parsing) and the download happy path.
|
||||
COMPOSE_PROFILES=
|
||||
SM_AA_URL=http://mock-aa
|
||||
SM_USE_CF_BYPASS=false
|
||||
SM_PROXY_MODE=none
|
||||
E2E_PROFILE=baseline
|
||||
@@ -0,0 +1,9 @@
|
||||
# Profile: bypasser-disabled (cluster 1 negative control)
|
||||
# AA behind the Cloudflare gate but the bypasser is OFF. Proves the gate really
|
||||
# challenges: search must FAIL clearly here, which is what makes the
|
||||
# bypasser-external success meaningful (and guards #202/#410 "uses AA when CF
|
||||
# bypass disabled" / "config not adhered").
|
||||
COMPOSE_PROFILES=bypasser-external
|
||||
SM_AA_URL=http://cf.mock.test
|
||||
SM_USE_CF_BYPASS=false
|
||||
E2E_PROFILE=bypasser-disabled
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Profile: bypasser-external (cluster 1)
|
||||
# AA is served behind a Cloudflare gate. The app must detect the challenge and
|
||||
# route through the external bypasser (mock FlareSolverr), which returns solved
|
||||
# HTML + cf_clearance. Regression surface: #284 #226 #202 #1030 #410 #369 #267.
|
||||
COMPOSE_PROFILES=bypasser-external
|
||||
SM_AA_URL=http://cf.mock.test
|
||||
SM_USE_CF_BYPASS=true
|
||||
# Must be set explicitly — shelfmark does NOT derive it from EXT_BYPASSER_URL.
|
||||
# Without it the app falls back to the in-image Chrome bypasser and never calls
|
||||
# FlareSolverr, so this profile would silently exercise the wrong path.
|
||||
SM_USING_EXTERNAL_BYPASSER=true
|
||||
SM_EXT_BYPASSER_URL=http://mock-flaresolverr
|
||||
E2E_PROFILE=bypasser-external
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Profile: client-deluge (cluster 5 — real Deluge client)
|
||||
# shelfmark talks to deluge-web (default WebUI password "deluge"), which connects
|
||||
# to the bundled daemon — no daemon auth-file juggling.
|
||||
COMPOSE_PROFILES=client-deluge
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=deluge
|
||||
SM_DELUGE_HOST=deluge
|
||||
SM_DELUGE_PORT=8112
|
||||
SM_DELUGE_PASSWORD=deluge
|
||||
E2E_DOWNLOAD_TIMEOUT=180
|
||||
E2E_PROFILE=client-deluge
|
||||
@@ -0,0 +1,21 @@
|
||||
# Profile: client-qbittorrent-delayed (cluster 5 — delayed completed-path visibility)
|
||||
# Same mock Prowlarr + webseed torrent as `full`, but qBittorrent writes to a
|
||||
# directory Shelfmark cannot see until the delayed sync sidecar copies it over.
|
||||
# This models seedbox / remote-sync setups where the client reports completion
|
||||
# before the completed path exists inside Shelfmark.
|
||||
COMPOSE_PROFILES=client-qbittorrent-delayed
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=qbittorrent
|
||||
SM_QBITTORRENT_URL=http://qbittorrent:8080
|
||||
SM_QBITTORRENT_USERNAME=admin
|
||||
SM_QBITTORRENT_PASSWORD=adminadmin
|
||||
SM_QBITTORRENT_CATEGORY=books
|
||||
|
||||
# The sync delay must be longer than the default client path polling interval
|
||||
# so Shelfmark actually enters the completed-path wait loop.
|
||||
SM_DELAYED_DOWNLOAD_SYNC_SECONDS=20
|
||||
SM_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT=90
|
||||
E2E_DOWNLOAD_TIMEOUT=180
|
||||
E2E_PROFILE=client-qbittorrent-delayed
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Profile: client-transmission (cluster 5 — real Transmission client)
|
||||
# Same mock Prowlarr + webseed torrent as `full`, but the torrent goes to a real
|
||||
# Transmission. Lean (no Chrome/CF/DoH), so it boots fast.
|
||||
COMPOSE_PROFILES=client-transmission
|
||||
SM_PROWLARR_ENABLED=true
|
||||
SM_PROWLARR_URL=http://mock-prowlarr
|
||||
SM_PROWLARR_API_KEY=e2e-test-key
|
||||
SM_PROWLARR_TORRENT_CLIENT=transmission
|
||||
SM_TRANSMISSION_URL=http://transmission:9091
|
||||
SM_TRANSMISSION_USERNAME=admin
|
||||
SM_TRANSMISSION_PASSWORD=admin
|
||||
E2E_DOWNLOAD_TIMEOUT=180
|
||||
E2E_PROFILE=client-transmission
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
# Profile: dns-blocked (config cluster: ISP DNS block — #1028 regression)
|
||||
# The container's *system* resolver (coredns-blocked) NXDOMAINs the AA host,
|
||||
# but the app's *custom* DNS resolver (coredns) resolves it fine. A correct app
|
||||
# reaches AA via custom DNS; a regressed one (system DNS in subprocess) fails.
|
||||
COMPOSE_PROFILES=dns-manual,dns-blocked
|
||||
SM_AA_URL=http://aa.mock.test
|
||||
SM_SYSTEM_DNS=172.30.0.22
|
||||
SM_CUSTOM_DNS=manual
|
||||
SM_CUSTOM_DNS_MANUAL=172.30.0.20
|
||||
E2E_PROFILE=dns-blocked
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user