Fix: HTTP grab behavior, logging enforcement (#521)

- Base URL option in settings for reverse proxy setups
- Fix NZB downloads not deleting on completion
- Fix handling for audiobook files over 100+ parts
- Fix prowlarr search timeout 
- Fix prowlarr categorisation for expanded searches

.github/workflows/build-and-publish-docker-image.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository_owner }}/shelfmark
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -20,15 +20,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - suffix: ""
-            target: cwa-bd
-            image_name_suffix: ""
-          - suffix: "-tor"
-            target: cwa-bd-tor
-            image_name_suffix: "-tor"
-          - suffix: "-extbp"
-            target: cwa-bd-extbp
-            image_name_suffix: "-extbp"
+          - target: shelfmark
+          - target: shelfmark-lite
+            image_name_suffix: "-lite"
     steps:
       - name: Get current date
         id: date
@@ -80,4 +74,71 @@ jobs:
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
+
+  # Create aliases for backwards compatibility
+  create-aliases:
+    needs: build-and-push-images
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    permissions:
+      contents: read
+      packages: write
+    env:
+      # Legacy name for backwards compatibility (hardcoded so it works after rename)
+      LEGACY_NAME: calibre-web-automated-book-downloader
+    steps:
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create legacy aliases
+        run: |
+          # Current image names (follows repo name)
+          STANDARD="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          LITE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-lite"
+
+          # Legacy image names (hardcoded for backwards compatibility)
+          LEGACY="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.LEGACY_NAME }}"
+          LEGACY_TOR="${LEGACY}-tor"
+          LEGACY_EXTBP="${LEGACY}-extbp"
+
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+
+          # Helper function to create alias with all standard tags
+          create_alias() {
+            local SOURCE=$1
+            local ALIAS=$2
+
+            # Always create SHA tag
+            docker buildx imagetools create -t "${ALIAS}:sha-${SHA_SHORT}" "${SOURCE}:sha-${SHA_SHORT}"
+
+            if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+              VERSION="${{ github.ref_name }}"
+              VERSION_NUM="${VERSION#v}"
+              MINOR="${VERSION_NUM%.*}"
+
+              docker buildx imagetools create -t "${ALIAS}:latest" "${SOURCE}:latest"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION_NUM}" "${SOURCE}:${VERSION_NUM}"
+              docker buildx imagetools create -t "${ALIAS}:${MINOR}" "${SOURCE}:${MINOR}"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION}" "${SOURCE}:${VERSION}"
+            else
+              docker buildx imagetools create -t "${ALIAS}:dev" "${SOURCE}:dev"
+            fi
+          }
+
+          # Create legacy aliases pointing to current images
+          # calibre-web-automated-book-downloader → standard image
+          create_alias "${STANDARD}" "${LEGACY}"
+
+          # calibre-web-automated-book-downloader-tor → standard image
+          create_alias "${STANDARD}" "${LEGACY_TOR}"
+
+          # calibre-web-automated-book-downloader-extbp → lite image
+          create_alias "${LITE}" "${LEGACY_EXTBP}"

.gitignore

@@ -228,3 +228,7 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python
 /downloaded_files
 /.local/
+*.local.*
+AGENTS.md
+.claude/
+.playwright-mcp/

.vscode/launch.json

@@ -2,17 +2,17 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Current CWABD File",
+      "name": "Python Debugger: Current Shelfmark File",
       "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false,
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader",
-        "TEMP_DIR": "/tmp/cwa-book-downloader",
+        "INGEST_DIR": "/tmp/shelfmark",
+        "TEMP_DIR": "/tmp/shelfmark",
         "LOG_LEVEL": "DEBUG",
-        "LOG_ROOT": "/tmp/cwa-book-downloader",
+        "LOG_ROOT": "/tmp/shelfmark",
         "ENABLE_LOGGING": "true",
         "DOCKERMODE": "false",
         "DEBUG": "true",
@@ -27,7 +27,7 @@
       "preLaunchTask": "docker-compose up (dev)", // Spin up dev containers
       "postDebugTask": "docker-compose down (dev)", // Optional: tear them down
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -38,7 +38,7 @@
       "preLaunchTask": "docker-compose up (prod)",
       "postDebugTask": "docker-compose down (prod)",
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -54,9 +54,9 @@
   ],
   "compounds": [
     {
-      "name": "Launch CWA-BD",
+      "name": "Launch Shelfmark",
       "configurations": [
-        "Launch cwa-bd app.py",
+        "Launch Shelfmark app.py",
         "Launch Browser"
       ]
     }

Dockerfile

@@ -14,8 +14,9 @@ WORKDIR /frontend
 # Copy frontend package files
 COPY src/frontend/package*.json ./
 
-# Install dependencies
-RUN npm ci
+# Install dependencies (cache mount for faster rebuilds)
+RUN --mount=type=cache,target=/root/.npm \
+    npm ci
 
 # Copy frontend source
 COPY src/frontend/ ./
@@ -44,9 +45,9 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    NAME=Calibre-Web-Automated-Book-Downloader \
+    NAME=Shelfmark \
     PYTHONPATH=/app \
-    # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
+    # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
     LC_ALL=en_US.UTF-8
@@ -67,7 +68,14 @@ RUN apt-get update && \
     # For debug
     zip iputils-ping \
     # For user switching
-    sudo && \
+    sudo \
+    # --- Tor support (activated via USING_TOR=true) ---
+    tor \
+    supervisor \
+    iptables && \
+    # Configure iptables alternatives for tor.sh compatibility
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
     # Cleanup APT cache *after* all installs in this layer
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
@@ -84,12 +92,11 @@ RUN apt-get update && \
 WORKDIR /app
 
 # Install Python dependencies using pip
-# Upgrade pip first, then copy requirements and install
-# Copying requirements-base.txt separately leverages build cache
-COPY requirements-base.txt .
-RUN pip install --no-cache-dir -r requirements-base.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
+# Copying requirements files separately leverages build cache
+# Cache mount persists pip cache between builds for faster installs
+COPY requirements-base.txt requirements-shelfmark.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-base.txt
 
 # Copy application code *after* dependencies are installed
 COPY . .
@@ -100,7 +107,7 @@ COPY --from=frontend-builder /frontend/dist /app/frontend-dist
 # Final setup: permissions and directories in one layer
 # Only creating directories and setting executable bits.
 # Ownership will be handled by the entrypoint script.
-RUN mkdir -p /var/log/cwa-book-downloader /cwa-book-ingest && \
+RUN mkdir -p /var/log/shelfmark /books && \
     chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
 
 # Expose the application port
@@ -115,7 +122,7 @@ HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 
-FROM base AS cwa-bd
+FROM base AS shelfmark
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -132,46 +139,25 @@ RUN apt-get update && \
     # For RAR extraction
     unrar-free && \
     # Create symlink so rarfile library can find unrar
-    ln -sf /usr/bin/unrar-free /usr/bin/unrar
-
-# install additional dependencies
-COPY requirements-cwa-bd.txt ./
-RUN pip install --no-cache-dir -r requirements-cwa-bd.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
-
-# Add this line to grant read/execute permissions to others
-RUN chmod -R o+rx /usr/bin/chromium && \
-    chmod -R o+rx /usr/bin/chromedriver && \
-    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
-
-# Default command to run the application entrypoint script
-CMD ["/app/entrypoint.sh"]
-
-FROM cwa-bd AS cwa-bd-tor
-
-ENV USING_TOR=true
-
-# Install Tor and dependencies
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    # --- Tor ---
-    tor \
-    # --- Supervisor ---
-    supervisor \
-    # --- iptables ---
-    iptables && \
-    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
-    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
-    # Cleanup APT cache *after* all installs in this layer
+    ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
+    # Cleanup APT cache
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Override the default command to run Tor
+# Install additional dependencies (requirements file already copied in base stage)
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-shelfmark.txt
+
+# Grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium && \
+    chmod -R o+rx /usr/bin/chromedriver && \
+    chmod -R o+rwx /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+
+# Default command to run the application entrypoint script
 CMD ["/app/entrypoint.sh"]
 
-FROM base AS cwa-bd-extbp
+FROM base AS shelfmark-lite
 
 ENV USING_EXTERNAL_BYPASSER=true
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install dev build preview typecheck clean up down docker-build refresh
+.PHONY: help install dev build preview typecheck clean up down docker-build refresh restart
 
 # Frontend directory
 FRONTEND_DIR := src/frontend
@@ -21,6 +21,7 @@ help:
 	@echo "Backend (Docker):"
 	@echo "  up         - Start backend services"
 	@echo "  down       - Stop backend services"
+	@echo "  restart    - Restart backend services (no rebuild)"
 	@echo "  docker-build - Build Docker image"
 	@echo "  refresh    - Rebuild and restart backend services"
 
@@ -70,6 +71,11 @@ docker-build:
 	@echo "Building Docker image..."
 	docker compose -f $(COMPOSE_FILE) build
 
+# Restart backend services (no rebuild)
+restart:
+	@echo "Restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) restart
+
 # Rebuild and restart backend services
 refresh:
 	@echo "Rebuilding and restarting backend services..."

README.v2.md

@@ -1,240 +0,0 @@
-# 📚 Book Downloader
-*calibre-web-automated-book-downloader*
-
-<img src="src/frontend/public/logo.png" alt="Book Downloader" width="200">
-
-A unified web interface for searching and downloading books from multiple sources — all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library.
-
-**Fully standalone** — no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) or [Booklore](https://github.com/booklore-app/booklore) for automatic import.
-
-## ✨ Features
-
-- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
-- **Real-Time Progress** - Unified download queue with live status updates across all sources
-- **Two Search Modes**:
-  - **Direct Download** - Search and download from popular web sources
-  - **Universal Mode** - Search metadata providers (Hardcover, Open Library) for richer book discovery and multi-source downloads *(additional sources in development - coming soon!)*
-- **Format Support** - EPUB, MOBI, AZW3, FB2, DJVU, CBZ, CBR and more
-- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
-- **PWA Support** - Install as a mobile app for quick access
-- **Docker Deployment** - Up and running in minutes
-
-## 🖼️ Screenshots
-
-**Home screen**
-![Home screen](README_images/homescreen.png 'Home screen')
-
-**Search results**
-![Search results](README_images/search-results.png 'Search results')
-
-**Multi-source downloads**
-![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
-
-**Download queue**
-![Download queue](README_images/downloads.png 'Download queue')
-
-## 🚀 Quick Start
-
-### Prerequisites
-
-- Docker & Docker Compose
-
-### Installation
-
-1. Download the docker-compose file:
-   ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.yml
-   ```
-
-2. Start the service:
-   ```bash
-   docker compose up -d
-   ```
-
-3. Open `http://localhost:8084`
-
-That's it! Configure settings through the web interface as needed.
-
-### Volume Setup
-
-```yaml
-volumes:
-  - /your/config/path:/config             # Config, database, and artwork cache directory
-  - /your/download/path:/cwa-book-ingest  # Downloaded books
-```
-
-> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
-
-> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
-
-## ⚙️ Configuration
-
-### Search Modes
-
-**Direct Download Mode** (default)
-- Works out of the box, no setup required
-- Searches a huge library of books directly
-- Returns downloadable releases immediately
-
-**Universal Mode**
-- Cleaner search results via metadata providers (Hardcover, Open Library)
-- Aggregates releases from multiple configured sources
-- Requires manual setup (API keys, additional sources)
-
-Set the mode via Settings or `SEARCH_MODE` environment variable.
-
-### Environment Variables
-
-Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
-
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `FLASK_PORT` | Web interface port | `8084` |
-| `INGEST_DIR` | Book download directory | `/cwa-book-ingest` |
-| `TZ` | Container timezone | `UTC` |
-| `UID` / `GID` | Runtime user/group ID | `1000` / `100` |
-| `SEARCH_MODE` | `direct` or `universal` | `direct` |
-
-Some of the additional options available in Settings:
-- **AA Donator Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
-- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
-- **Content Folders** - Route fiction, non-fiction, comics, etc. to separate directories
-- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable
-- **Format & Language** - Filter downloads by preferred formats and languages
-- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
-
-## 🐳 Docker Variants
-
-### Standard
-```bash
-docker compose up -d
-```
-
-### Tor Variant
-Routes all traffic through Tor for enhanced privacy:
-```bash
-curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.tor.yml
-docker compose -f docker-compose.tor.yml up -d
-```
-
-**Notes:**
-- Requires `NET_ADMIN` and `NET_RAW` capabilities
-- Timezone is auto-detected from Tor exit node
-- Custom DNS/proxy settings are ignored
-
-### External Cloudflare Resolver
-Use FlareSolverr or ByParr instead of the built-in bypasser:
-```bash
-curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.extbp.yml
-docker compose -f docker-compose.extbp.yml up -d
-```
-
-Configure the resolver URL in Settings under the Cloudflare tab.
-
-**When to use external vs internal bypasser:**
-- **External** is useful if you already run FlareSolverr for other services (saves resources) or if you rarely need bypassing
-- **Internal** (default) is faster and more reliable for most users - it's optimized specifically for this application
-
-## 🔐 Authentication
-
-Authentication is optional but recommended for shared or exposed instances. Enable in Settings.
-
-**Alternative**: If you're running Calibre-Web, you can reuse its user database by mounting it:
-
-```yaml
-volumes:
-  - /path/to/calibre-web/app.db:/auth/app.db:ro
-```
-
-## Health Monitoring
-
-The application exposes a health endpoint at `/api/status`. Add a health check to your compose:
-
-```yaml
-healthcheck:
-  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/status"]
-  interval: 30s
-  timeout: 30s
-  retries: 3
-```
-
-## Logging
-
-Logs are available via:
-- `docker logs <container-name>`
-- `/var/log/cwa-book-downloader/` inside the container (when `ENABLE_LOGGING=true`)
-
-Log level is configurable via Settings or `LOG_LEVEL` environment variable.
-
-## Development
-
-```bash
-# Frontend development
-make install     # Install dependencies
-make dev         # Start Vite dev server (localhost:5173)
-make build       # Production build
-make typecheck   # TypeScript checks
-
-# Backend (Docker)
-make up          # Start backend via docker-compose.dev.yml
-make down        # Stop services
-make refresh     # Rebuild and restart
-```
-
-The frontend dev server proxies to the backend on port 8084.
-
-### Architecture
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│                      Web Interface                          │
-│                 (React + TypeScript + Vite)                 │
-├─────────────────────────────────────────────────────────────┤
-│                      Flask Backend                          │
-│                   (REST API + WebSocket)                    │
-├───────────────────┬─────────────────────┬───────────────────┤
-│ Metadata Providers│   Download Queue    │  Cloudflare       │
-│                   │   & Orchestrator    │  Bypass           │
-├───────────────────┼─────────────────────┼───────────────────┤
-│ • Hardcover       │ • Task scheduling   │ • Internal        │
-│ • Open Library    │ • Progress tracking │ • External        │
-│                   │ • Retry logic       │   (FlareSolverr)  │
-├───────────────────┴─────────────────────┴───────────────────┤
-│                     Release Sources                         │
-├─────────────────────────────────────────────────────────────┤
-│ • Direct Download (Anna's Archive → Libgen → Welib)         │
-├─────────────────────────────────────────────────────────────┤
-│                     Network Layer                           │
-├─────────────────────────────────────────────────────────────┤
-│ • Auto DNS rotation  • Mirror failover  • Resume support    │
-└─────────────────────────────────────────────────────────────┘
-```
-
-The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
-
-## Contributing
-
-Contributions are welcome! Please file issues or submit pull requests on GitHub.
-
-> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR!
-
-## License
-
-MIT License - see [LICENSE](LICENSE) for details.
-
-## ⚠️ Disclaimers
-
-### Copyright Notice
-
-This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
-- Ensuring they have the right to download requested materials
-- Respecting copyright laws and intellectual property rights
-- Using the tool in compliance with their local regulations
-
-### Library Integration
-
-Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
-
-## Support
-
-For issues or questions, please [file an issue](https://github.com/calibrain/calibre-web-automated-book-downloader/issues) on GitHub.

compose/docker-compose.lite.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark-lite:
+    image: ghcr.io/calibrain/shelfmark-lite:latest
+    environment:
+      # EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.tor.yml

@@ -0,0 +1,20 @@
+# Routes all traffic through Tor - requires NET_ADMIN capability
+services:
+  shelfmark-tor:
+    image: ghcr.io/calibrain/shelfmark:latest
+    environment:
+      FLASK_PORT: 8084
+      USING_TOR: true
+      PUID: 1000
+      PGID: 1000
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    container_name: shelfmark
+    environment:
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

cwa_book_downloader/__init__.py

@@ -1 +0,0 @@
-"""CWA Book Downloader - book search and download service."""

cwa_book_downloader/__main__.py

@@ -1,8 +0,0 @@
-"""Package entry point for `python -m cwa_book_downloader`."""
-
-from cwa_book_downloader.main import app, socketio
-from cwa_book_downloader.config.env import FLASK_HOST, FLASK_PORT
-from cwa_book_downloader.core.config import config
-
-if __name__ == "__main__":
-    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

cwa_book_downloader/bypass/__init__.py

@@ -1 +0,0 @@
-"""Cloudflare bypass utilities."""

cwa_book_downloader/bypass/external_bypasser.py

@@ -1,162 +0,0 @@
-"""External Cloudflare bypasser using FlareSolverr."""
-
-from threading import Event
-from typing import Optional, TYPE_CHECKING
-import requests
-import time
-import random
-
-from cwa_book_downloader.core.config import config
-from cwa_book_downloader.core.logger import setup_logger
-
-if TYPE_CHECKING:
-    from cwa_book_downloader.download import network
-
-
-class BypassCancelledException(Exception):
-    """Raised when a bypass operation is cancelled."""
-    pass
-
-logger = setup_logger(__name__)
-
-# Connection timeout (seconds) - how long to wait for external bypasser to accept connection
-CONNECT_TIMEOUT = 10
-# Maximum read timeout cap (seconds) - hard limit regardless of EXT_BYPASSER_TIMEOUT
-MAX_READ_TIMEOUT = 120
-# Buffer added to bypasser's configured timeout (seconds) - accounts for processing overhead
-READ_TIMEOUT_BUFFER = 15
-# Retry settings for bypasser failures
-MAX_RETRY = 5
-BACKOFF_BASE = 1.0
-BACKOFF_CAP = 10.0
-
-
-def _fetch_via_bypasser(target_url: str) -> Optional[str]:
-    """Make a single request to the external bypasser service.
-
-    Args:
-        target_url: The URL to fetch through the bypasser
-
-    Returns:
-        HTML content if successful, None otherwise
-    """
-    bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
-    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
-    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
-
-    if not bypasser_url or not bypasser_path:
-        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
-        return None
-
-    bypasser_endpoint = f"{bypasser_url}{bypasser_path}"
-    headers = {"Content-Type": "application/json"}
-    payload = {
-        "cmd": "request.get",
-        "url": target_url,
-        "maxTimeout": bypasser_timeout
-    }
-
-    # Calculate read timeout: bypasser timeout (ms -> s) + buffer, capped at max
-    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
-
-    try:
-        response = requests.post(
-            bypasser_endpoint,
-            headers=headers,
-            json=payload,
-            timeout=(CONNECT_TIMEOUT, read_timeout)
-        )
-        response.raise_for_status()
-        result = response.json()
-
-        status = result.get('status', 'unknown')
-        message = result.get('message', '')
-        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
-
-        # Check for error status (bypasser returns status="error" with solution=null on failure)
-        if status != 'ok':
-            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
-            return None
-
-        solution = result.get('solution')
-        if not solution:
-            logger.warning(f"External bypasser returned empty solution for '{target_url}'")
-            return None
-
-        html = solution.get('response', '')
-        if not html:
-            logger.warning(f"External bypasser returned empty response for '{target_url}'")
-            return None
-
-        return html
-
-    except requests.exceptions.Timeout:
-        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
-        return None
-    except requests.exceptions.RequestException as e:
-        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
-        return None
-    except (KeyError, TypeError, ValueError) as e:
-        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
-        return None
-
-
-def get_bypassed_page(url: str, selector: Optional["network.AAMirrorSelector"] = None, cancel_flag: Optional[Event] = None) -> Optional[str]:
-    """Fetch HTML content from a URL using an external Cloudflare bypasser service.
-
-    Retries with exponential backoff and mirror/DNS rotation on failure.
-
-    Args:
-        url: Target URL to fetch
-        selector: Mirror selector for AA URL rewriting and rotation
-        cancel_flag: Optional threading Event to signal cancellation
-
-    Returns:
-        HTML content if successful, None otherwise
-
-    Raises:
-        BypassCancelledException: If cancel_flag is set during operation
-    """
-    from cwa_book_downloader.download import network as network_module
-    sel = selector or network_module.AAMirrorSelector()
-
-    for attempt in range(1, MAX_RETRY + 1):
-        # Check for cancellation before each attempt
-        if cancel_flag and cancel_flag.is_set():
-            logger.info("External bypasser cancelled by user")
-            raise BypassCancelledException("Bypass cancelled")
-
-        attempt_url = sel.rewrite(url)
-        result = _fetch_via_bypasser(attempt_url)
-        if result:
-            return result
-
-        if attempt == MAX_RETRY:
-            break
-
-        # Check for cancellation before backoff wait
-        if cancel_flag and cancel_flag.is_set():
-            logger.info("External bypasser cancelled during retry")
-            raise BypassCancelledException("Bypass cancelled")
-
-        # Backoff with jitter before retry, checking cancellation during wait
-        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
-        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
-
-        # Check cancellation during delay (check every second)
-        for _ in range(int(delay)):
-            if cancel_flag and cancel_flag.is_set():
-                logger.info("External bypasser cancelled during backoff")
-                raise BypassCancelledException("Bypass cancelled")
-            time.sleep(1)
-        # Sleep remaining fraction
-        remaining = delay - int(delay)
-        if remaining > 0:
-            time.sleep(remaining)
-
-        # Rotate mirror/DNS for next attempt
-        new_base, action = sel.next_mirror_or_rotate_dns()
-        if action in ("mirror", "dns") and new_base:
-            logger.info(f"Rotated {action} for retry")
-
-    return None

cwa_book_downloader/config/env.py

@@ -1,151 +0,0 @@
-"""Environment variable parsing. No local dependencies - import first."""
-
-import os
-import shutil
-from pathlib import Path
-
-
-def string_to_bool(s: str) -> bool:
-    return s.lower() in ["true", "yes", "1", "y"]
-
-
-# Authentication and session settings
-SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
-
-CWA_DB = os.getenv("CWA_DB_PATH")
-CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
-CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
-LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
-LOG_DIR = LOG_ROOT / "cwa-book-downloader"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
-INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
-INGEST_DIR_BOOK_FICTION = os.getenv("INGEST_DIR_BOOK_FICTION", "")
-INGEST_DIR_BOOK_NON_FICTION = os.getenv("INGEST_DIR_BOOK_NON_FICTION", "")
-INGEST_DIR_BOOK_UNKNOWN = os.getenv("INGEST_DIR_BOOK_UNKNOWN", "")
-INGEST_DIR_MAGAZINE = os.getenv("INGEST_DIR_MAGAZINE", "")
-INGEST_DIR_COMIC_BOOK = os.getenv("INGEST_DIR_COMIC_BOOK", "")
-INGEST_DIR_AUDIOBOOK = os.getenv("INGEST_DIR_AUDIOBOOK", "")
-INGEST_DIR_STANDARDS_DOCUMENT = os.getenv("INGEST_DIR_STANDARDS_DOCUMENT", "")
-INGEST_DIR_MUSICAL_SCORE = os.getenv("INGEST_DIR_MUSICAL_SCORE", "")
-INGEST_DIR_OTHER = os.getenv("INGEST_DIR_OTHER", "")
-DOWNLOAD_PATHS = {
-    "book (fiction)": Path(INGEST_DIR_BOOK_FICTION) if INGEST_DIR_BOOK_FICTION else INGEST_DIR,
-    "book (non-fiction)": Path(INGEST_DIR_BOOK_NON_FICTION) if INGEST_DIR_BOOK_NON_FICTION else INGEST_DIR,
-    "book (unknown)": Path(INGEST_DIR_BOOK_UNKNOWN) if INGEST_DIR_BOOK_UNKNOWN else INGEST_DIR,
-    "magazine": Path(INGEST_DIR_MAGAZINE) if INGEST_DIR_MAGAZINE else INGEST_DIR,
-    "comic book": Path(INGEST_DIR_COMIC_BOOK) if INGEST_DIR_COMIC_BOOK else INGEST_DIR,
-    "audiobook": Path(INGEST_DIR_AUDIOBOOK) if INGEST_DIR_AUDIOBOOK else INGEST_DIR,
-    "standards document": Path(INGEST_DIR_STANDARDS_DOCUMENT) if INGEST_DIR_STANDARDS_DOCUMENT else INGEST_DIR,
-    "musical score": Path(INGEST_DIR_MUSICAL_SCORE) if INGEST_DIR_MUSICAL_SCORE else INGEST_DIR,
-    "other": Path(INGEST_DIR_OTHER) if INGEST_DIR_OTHER else INGEST_DIR,
-}
-
-STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
-USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
-MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
-DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
-USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
-HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
-HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
-AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
-_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
-_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
-_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
-_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
-_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
-FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
-FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
-# Debug: skip specific download sources for testing fallback chains
-# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
-_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
-DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
-
-# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
-# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
-_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
-_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
-
-# Version information from Docker build
-BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
-RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
-
-# If debug is true, we want to log everything
-if DEBUG:
-    LOG_LEVEL = "DEBUG"
-else:
-    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
-MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
-MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1"))
-DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip()
-USE_DOH = string_to_bool(os.getenv("USE_DOH", "true"))
-BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
-BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true"))
-
-# Logging settings
-LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
-
-USING_EXTERNAL_BYPASSER = string_to_bool(os.getenv("USING_EXTERNAL_BYPASSER", "false"))
-if USING_EXTERNAL_BYPASSER:
-    EXT_BYPASSER_URL = os.getenv("EXT_BYPASSER_URL", "http://flaresolverr:8191").strip()
-    EXT_BYPASSER_PATH = os.getenv("EXT_BYPASSER_PATH", "/v1").strip()
-    EXT_BYPASSER_TIMEOUT = int(os.getenv("EXT_BYPASSER_TIMEOUT", "60000"))
-
-USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
-# If using Tor, we don't need to set custom DNS, use DOH, or proxy
-if USING_TOR:
-    _CUSTOM_DNS = ""
-    USE_DOH = False
-    HTTP_PROXY = ""
-    HTTPS_PROXY = ""
-
-# Detect Tor variant (has tor binary installed)
-TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
-
-# Calibre-Web URL for navigation button
-CALIBRE_WEB_URL = os.getenv("CALIBRE_WEB_URL", "").strip()
-
-# Metadata provider settings (Stage 2)
-# Set to "hardcover" or "openlibrary" to enable metadata-first search mode
-METADATA_PROVIDER = os.getenv("METADATA_PROVIDER", "").strip().lower()
-HARDCOVER_API_KEY = os.getenv("HARDCOVER_API_KEY", "").strip()
-
-# Cache TTL settings (in seconds)
-METADATA_CACHE_SEARCH_TTL = int(os.getenv("METADATA_CACHE_SEARCH_TTL", "300"))  # 5 minutes
-METADATA_CACHE_BOOK_TTL = int(os.getenv("METADATA_CACHE_BOOK_TTL", "600"))  # 10 minutes
-
-# Cover image cache settings
-def _is_config_dir_writable() -> bool:
-    """Check if the config directory exists and is writable."""
-    try:
-        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
-            return False
-        test_file = CONFIG_DIR / ".write_test"
-        test_file.touch()
-        test_file.unlink()
-        return True
-    except (OSError, PermissionError):
-        return False
-
-
-def is_covers_cache_enabled() -> bool:
-    """Check if cover caching is enabled (dynamic, respects settings changes).
-
-    Cache is only enabled if:
-    1. The COVERS_CACHE_ENABLED setting is true
-    2. The config directory is writable
-    """
-    from cwa_book_downloader.core.config import config
-    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
-    return setting_enabled and _is_config_dir_writable()
-
-
-# Legacy static value - use is_covers_cache_enabled() for dynamic checks
-_COVERS_CACHE_ENABLED_ENV = string_to_bool(os.getenv("COVERS_CACHE_ENABLED", "true"))
-COVERS_CACHE_ENABLED = _COVERS_CACHE_ENABLED_ENV and _is_config_dir_writable()
-COVERS_CACHE_DIR = CONFIG_DIR / "covers"
-COVERS_CACHE_TTL = int(os.getenv("COVERS_CACHE_TTL", "0"))  # 0 = forever (covers are static)
-COVERS_CACHE_MAX_SIZE_MB = int(os.getenv("COVERS_CACHE_MAX_SIZE_MB", "500"))

cwa_book_downloader/config/security.py

@@ -1,152 +0,0 @@
-"""Authentication settings registration."""
-
-from typing import Any, Dict
-
-from werkzeug.security import generate_password_hash
-
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.settings_registry import (
-    register_settings,
-    register_on_save,
-    load_config_file,
-    TextField,
-    PasswordField,
-    CheckboxField,
-    ActionButton,
-)
-
-logger = setup_logger(__name__)
-
-
-def _clear_builtin_credentials() -> Dict[str, Any]:
-    """Clear built-in credentials to allow public access."""
-    try:
-        config = load_config_file("security")
-        config.pop("BUILTIN_USERNAME", None)
-        config.pop("BUILTIN_PASSWORD_HASH", None)
-
-        # Save the cleared config
-        from cwa_book_downloader.core.settings_registry import _get_config_file_path, _ensure_config_dir
-        import json
-
-        _ensure_config_dir("security")
-        config_path = _get_config_file_path("security")
-        with open(config_path, 'w') as f:
-            json.dump(config, f, indent=2)
-
-        logger.info("Cleared credentials")
-        return {"success": True, "message": "Credentials cleared. The app is now publicly accessible."}
-
-    except Exception as e:
-        logger.error(f"Failed to clear credentials: {e}")
-        return {"success": False, "message": f"Failed to clear credentials: {str(e)}"}
-
-
-def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Custom save handler for security settings.
-
-    Handles password validation and hashing:
-    - If new password is provided, validate confirmation and hash it
-    - If password fields are empty, preserve existing hash
-    - Never store raw passwords
-
-    Returns:
-        Dict with processed values to save and any validation errors.
-    """
-    password = values.get("BUILTIN_PASSWORD", "")
-    password_confirm = values.get("BUILTIN_PASSWORD_CONFIRM", "")
-
-    # Remove raw password fields - they should never be persisted
-    values.pop("BUILTIN_PASSWORD", None)
-    values.pop("BUILTIN_PASSWORD_CONFIRM", None)
-
-    # If password is provided, validate and hash it
-    if password:
-        if password != password_confirm:
-            return {
-                "error": True,
-                "message": "Passwords do not match",
-                "values": values
-            }
-
-        if len(password) < 4:
-            return {
-                "error": True,
-                "message": "Password must be at least 4 characters",
-                "values": values
-            }
-
-        # Hash the password
-        values["BUILTIN_PASSWORD_HASH"] = generate_password_hash(password)
-        logger.info("Password hash updated")
-
-    # If no password provided but username is being set, preserve existing hash
-    elif "BUILTIN_USERNAME" in values:
-        existing = load_config_file("security")
-        if "BUILTIN_PASSWORD_HASH" in existing:
-            values["BUILTIN_PASSWORD_HASH"] = existing["BUILTIN_PASSWORD_HASH"]
-
-    return {"error": False, "values": values}
-
-
-@register_settings("security", "Security", icon="shield", order=5)
-def security_settings():
-    """Security and authentication settings."""
-    from cwa_book_downloader.config.env import CWA_DB_PATH
-    import os
-
-    cwa_db_available = CWA_DB_PATH and os.path.exists(CWA_DB_PATH)
-
-    fields = [
-        TextField(
-            key="BUILTIN_USERNAME",
-            label="Username",
-            description="Set a username and password to require login. Leave both empty for public access.",
-            placeholder="Enter username",
-            env_supported=False,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        PasswordField(
-            key="BUILTIN_PASSWORD",
-            label="Set Password",
-            description="Fill in to set or change the password.",
-            placeholder="Enter new password",
-            env_supported=False,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        PasswordField(
-            key="BUILTIN_PASSWORD_CONFIRM",
-            label="Confirm Password",
-            placeholder="Confirm new password",
-            env_supported=False,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        ActionButton(
-            key="clear_credentials",
-            label="Clear Credentials",
-            description="Remove login requirement and make the app publicly accessible.",
-            style="danger",
-            callback=_clear_builtin_credentials,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        CheckboxField(
-            key="USE_CWA_AUTH",
-            label="Use Calibre-Web Database",
-            description=(
-                "Authenticate using your existing Calibre-Web users instead of the credentials above."
-                if cwa_db_available
-                else "Authenticate using your existing Calibre-Web users. Set the CWA_DB_PATH environment variable to your Calibre-Web app.db file to enable this option."
-            ),
-            default=False,
-            env_supported=False,
-            disabled=not cwa_db_available,
-            disabled_reason="Set the CWA_DB_PATH environment variable to your Calibre-Web app.db file path to enable this option.",
-        ),
-    ]
-
-    return fields
-
-
-# Register the on_save handler for this tab
-register_on_save("security", _on_save_security)

cwa_book_downloader/config/settings.py

@@ -1,871 +0,0 @@
-"""Core settings registration and derived configuration values."""
-
-import os
-from pathlib import Path
-import json
-
-from cwa_book_downloader.config import env
-from cwa_book_downloader.core.logger import setup_logger
-
-logger = setup_logger(__name__)
-
-# Log configuration values at DEBUG level, filtering out module imports and functions
-logger.debug("Environment configuration:")
-for key, value in env.__dict__.items():
-    # Skip private attributes, modules, types, and callables (functions)
-    if key.startswith('_'):
-        continue
-    if isinstance(value, type) or callable(value):
-        continue
-    # Don't log module objects (they have __name__ attribute)
-    if hasattr(value, '__name__') and hasattr(value, '__file__'):
-        continue
-    # Redact sensitive values
-    if key == "AA_DONATOR_KEY" and isinstance(value, str) and value.strip():
-        value = "REDACTED"
-    if key == "HARDCOVER_API_KEY" and isinstance(value, str) and value.strip():
-        value = "REDACTED"
-    logger.debug(f"  {key}: {value}")
-
-# Load supported book languages from data file
-# Path is relative to the package root, not this file
-_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
-with open(_DATA_DIR / "book-languages.json") as file:
-    _SUPPORTED_BOOK_LANGUAGE = json.load(file)
-
-# Directory settings
-BASE_DIR = Path(__file__).resolve().parent.parent.parent
-logger.debug(f"BASE_DIR: {BASE_DIR}")
-if env.ENABLE_LOGGING:
-    env.LOG_DIR.mkdir(exist_ok=True)
-
-# Create necessary directories
-env.TMP_DIR.mkdir(exist_ok=True)
-env.INGEST_DIR.mkdir(exist_ok=True)
-
-CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
-logger.debug(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
-logger.debug(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
-logger.debug(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
-
-# DNS placeholders - actual values set by network.init() from config/ENV
-CUSTOM_DNS: list[str] = []
-DOH_SERVER: str = ""
-
-# Warn about external bypasser DNS limitations
-if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS:
-    logger.warning(
-        "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
-        "not this application's custom DNS settings. If you experience DNS-related blocks, "
-        "configure DNS at the Docker/system level for your FlareSolverr container, "
-        "or consider using the internal bypasser which integrates with the app's DNS system."
-    )
-
-# Proxy settings
-PROXIES = {}
-if env.HTTP_PROXY:
-    PROXIES["http"] = env.HTTP_PROXY
-if env.HTTPS_PROXY:
-    PROXIES["https"] = env.HTTPS_PROXY
-logger.debug(f"PROXIES: {PROXIES}")
-
-# Anna's Archive settings
-AA_BASE_URL = env._AA_BASE_URL
-AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
-AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
-AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
-
-# File format settings
-SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
-logger.debug(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
-
-# Complex language processing logic kept in config.py
-BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
-BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
-if len(BOOK_LANGUAGE) == 0:
-    BOOK_LANGUAGE = ['en']
-
-# Custom script settings with validation logic
-CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
-if CUSTOM_SCRIPT:
-    if not os.path.exists(CUSTOM_SCRIPT):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
-        CUSTOM_SCRIPT = ""
-    elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
-        CUSTOM_SCRIPT = ""
-
-# Debugging settings
-if not env.USING_EXTERNAL_BYPASSER:
-    # Virtual display settings for debugging internal cloudflare bypasser
-    VIRTUAL_SCREEN_SIZE = (1024, 768)
-    RECORDING_DIR = env.LOG_DIR / "recording"
-
-
-from cwa_book_downloader.core.settings_registry import (
-    register_settings,
-    register_group,
-    TextField,
-    PasswordField,
-    NumberField,
-    CheckboxField,
-    SelectField,
-    MultiSelectField,
-    OrderableListField,
-    HeadingField,
-    ActionButton,
-)
-
-
-register_group(
-    "direct_download",
-    "Anna's Archive",
-    icon="download",
-    order=20
-)
-
-register_group(
-    "metadata_providers",
-    "Metadata Providers",
-    icon="book",
-    order=12  # Between Network (10) and Advanced (15)
-)
-
-
-# Anna's Archive sort options (for Direct mode)
-_AA_SORT_OPTIONS = [
-    {"value": "relevance", "label": "Most relevant"},
-    {"value": "newest", "label": "Newest (publication year)"},
-    {"value": "oldest", "label": "Oldest (publication year)"},
-    {"value": "largest", "label": "Largest (filesize)"},
-    {"value": "smallest", "label": "Smallest (filesize)"},
-    {"value": "newest_added", "label": "Newest (open sourced)"},
-    {"value": "oldest_added", "label": "Oldest (open sourced)"},
-]
-
-_FORMAT_OPTIONS = [
-    {"value": "epub", "label": "EPUB"},
-    {"value": "mobi", "label": "MOBI"},
-    {"value": "azw3", "label": "AZW3"},
-    {"value": "pdf", "label": "PDF"},
-    {"value": "fb2", "label": "FB2"},
-    {"value": "djvu", "label": "DJVU"},
-    {"value": "cbz", "label": "CBZ"},
-    {"value": "cbr", "label": "CBR"},
-    {"value": "txt", "label": "TXT"},
-    {"value": "rtf", "label": "RTF"},
-    {"value": "doc", "label": "DOC"},
-    {"value": "docx", "label": "DOCX"},
-    {"value": "zip", "label": "ZIP"},
-    {"value": "rar", "label": "RAR"},
-]
-
-
-def _get_metadata_provider_options():
-    """Build metadata provider options dynamically from enabled providers only."""
-    from cwa_book_downloader.metadata_providers import list_providers, is_provider_enabled
-
-    options = []
-    for provider in list_providers():
-        # Only show providers that are enabled
-        if is_provider_enabled(provider["name"]):
-            options.append({"value": provider["name"], "label": provider["display_name"]})
-
-    # If no providers enabled, show a placeholder option
-    if not options:
-        options = [
-            {"value": "", "label": "No providers enabled"},
-        ]
-
-    return options
-
-
-def _get_release_source_options():
-    """Build release source options dynamically from registered sources."""
-    from cwa_book_downloader.release_sources import list_available_sources
-
-    return [
-        {"value": source["name"], "label": source["display_name"]}
-        for source in list_available_sources()
-    ]
-
-_LANGUAGE_OPTIONS = [{"value": lang["code"], "label": lang["language"]} for lang in _SUPPORTED_BOOK_LANGUAGE]
-
-
-def _clear_covers_cache(current_values: dict) -> dict:
-    """Clear the cover image cache."""
-    try:
-        from cwa_book_downloader.core.image_cache import get_image_cache, reset_image_cache
-
-        cache = get_image_cache()
-        count = cache.clear()
-
-        # Reset the singleton so it reinitializes with fresh state
-        reset_image_cache()
-
-        return {
-            "success": True,
-            "message": f"Cleared {count} cached cover images.",
-        }
-    except Exception as e:
-        logger.error(f"Failed to clear cover cache: {e}")
-        return {
-            "success": False,
-            "message": f"Failed to clear cache: {str(e)}",
-        }
-
-
-def _clear_metadata_cache(current_values: dict) -> dict:
-    """Clear the in-memory metadata cache."""
-    try:
-        from cwa_book_downloader.core.cache import get_metadata_cache
-
-        cache = get_metadata_cache()
-        stats_before = cache.stats()
-        cache.clear()
-
-        return {
-            "success": True,
-            "message": f"Cleared {stats_before['size']} cached entries.",
-        }
-    except Exception as e:
-        logger.error(f"Failed to clear metadata cache: {e}")
-        return {
-            "success": False,
-            "message": f"Failed to clear cache: {str(e)}",
-        }
-
-
-@register_settings("general", "General", icon="settings", order=0)
-def general_settings():
-    """Core application settings."""
-    return [
-        TextField(
-            key="CALIBRE_WEB_URL",
-            label="Book Management App URL",
-            description="Adds a navigation button to your book manager instance (Calibre-Web Automated, Booklore, etc).",
-            placeholder="http://calibre-web:8083",
-        ),
-        HeadingField(
-            key="search_mode_heading",
-            title="Search Mode",
-            description="Direct searches Anna's Archive and downloads immediately. Universal searches book metadata first, letting you choose from multiple release sources including Anna's Archive and Prowlarr.",
-        ),
-        SelectField(
-            key="SEARCH_MODE",
-            label="Search Mode",
-            description="How you want to search for and download books.",
-            options=[
-                {
-                    "value": "direct",
-                    "label": "Direct (Anna's Archive)",
-                    "description": "Search Anna's Archive and download directly. Works out of the box.",
-                },
-                {
-                    "value": "universal",
-                    "label": "Universal",
-                    "description": "Metadata-based search with downloads from all sources.",
-                },
-            ],
-            default="direct",
-        ),
-        SelectField(
-            key="AA_DEFAULT_SORT",
-            label="Default Sort Order",
-            description="Default sort order for Anna's Archive search results.",
-            options=_AA_SORT_OPTIONS,
-            default="relevance",
-            env_supported=False,  # UI-only setting
-            show_when={"field": "SEARCH_MODE", "value": "direct"},
-        ),
-        SelectField(
-            key="METADATA_PROVIDER",
-            label="Metadata Provider",
-            description="Choose which metadata provider to use for book searches.",
-            options=_get_metadata_provider_options,  # Callable - evaluated lazily to avoid circular imports
-            default="openlibrary",
-            show_when={"field": "SEARCH_MODE", "value": "universal"},
-        ),
-        SelectField(
-            key="DEFAULT_RELEASE_SOURCE",
-            label="Default Release Source",
-            description="The release source tab to open by default in the release modal.",
-            options=_get_release_source_options,  # Callable - evaluated lazily to avoid circular imports
-            default="direct_download",
-            env_supported=False,  # UI-only setting, not configurable via ENV
-            show_when={"field": "SEARCH_MODE", "value": "universal"},
-        ),
-        HeadingField(
-            key="search_defaults_heading",
-            title="Default Search Options",
-            description="Default filters applied to searches. Can be overridden using advanced search options.",
-        ),
-        MultiSelectField(
-            key="SUPPORTED_FORMATS",
-            label="Supported Formats",
-            description="Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found.",
-            options=_FORMAT_OPTIONS,
-            default=["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
-        ),
-        MultiSelectField(
-            key="BOOK_LANGUAGE",
-            label="Default Book Languages",
-            description="Default language filter for searches.",
-            options=_LANGUAGE_OPTIONS,
-            default=["en"],
-        ),
-    ]
-
-
-@register_settings("network", "Network", icon="globe", order=10)
-def network_settings():
-    """Network and connectivity settings."""
-    # Check if Tor variant is available and if Tor is currently enabled
-    tor_available = env.TOR_VARIANT_AVAILABLE
-    tor_enabled = env.USING_TOR
-
-    # When Tor is enabled (only possible in Tor variant), DNS/proxy settings are overridden
-    # The Tor variant uses iptables to force ALL traffic through Tor - it cannot be disabled
-    tor_overrides_network = tor_available  # If Tor variant, network settings are always managed by Tor
-
-    return [
-        SelectField(
-            key="CUSTOM_DNS",
-            label="DNS Provider",
-            description=(
-                "Managed by Tor when Tor routing is enabled."
-                if tor_overrides_network
-                else "DNS provider for domain resolution. 'Auto' rotates through providers on failure."
-            ),
-            options=[
-                {"value": "auto", "label": "Auto (Recommended)"},
-                {"value": "system", "label": "System"},
-                {"value": "google", "label": "Google"},
-                {"value": "cloudflare", "label": "Cloudflare"},
-                {"value": "quad9", "label": "Quad9"},
-                {"value": "opendns", "label": "OpenDNS"},
-                {"value": "manual", "label": "Manual"},
-            ],
-            default="auto",
-            disabled=tor_overrides_network,
-            disabled_reason="DNS is managed by Tor when Tor routing is enabled.",
-        ),
-        TextField(
-            key="CUSTOM_DNS_MANUAL",
-            label="Manual DNS Servers",
-            description="Comma-separated list of DNS server IP addresses (e.g., 8.8.8.8, 1.1.1.1).",
-            placeholder="8.8.8.8, 1.1.1.1",
-            disabled=tor_overrides_network,
-            disabled_reason="DNS is managed by Tor when Tor routing is enabled.",
-            show_when={"field": "CUSTOM_DNS", "value": "manual"},
-        ),
-        CheckboxField(
-            key="USE_DOH",
-            label="Use DNS over HTTPS",
-            description=(
-                "Not applicable when Tor routing is enabled."
-                if tor_overrides_network
-                else "Use encrypted DNS queries for improved reliability and privacy."
-            ),
-            default=True,
-            disabled=tor_overrides_network,
-            disabled_reason="DNS over HTTPS is not used when Tor routing is enabled.",
-            # Hide for manual and system (no DoH endpoint available for custom IPs or system DNS)
-            show_when={"field": "CUSTOM_DNS", "value": ["auto", "google", "cloudflare", "quad9", "opendns"]},
-            # Disable for auto (always uses DoH)
-            disabled_when={
-                "field": "CUSTOM_DNS",
-                "value": "auto",
-                "reason": "Auto mode always uses DNS over HTTPS for reliable provider rotation.",
-            },
-        ),
-        CheckboxField(
-            key="USING_TOR",
-            label="Tor Routing",
-            description=(
-                "All traffic is routed through Tor in this container variant. This cannot be changed."
-                if tor_available
-                else "Tor routing is not available in this container variant."
-            ),
-            default=tor_available,  # Reflects actual state: True if Tor variant, False otherwise
-            disabled=True,  # Always disabled - Tor state is determined by container variant
-            disabled_reason=(
-                "Tor routing is always active in the Tor container variant."
-                if tor_available
-                else "Requires the Tor container variant (calibre-web-automated-book-downloader-tor)."
-            ),
-        ),
-        SelectField(
-            key="PROXY_MODE",
-            label="Proxy Mode",
-            description=(
-                "Not applicable when Tor routing is enabled."
-                if tor_overrides_network
-                else "Choose proxy type. SOCKS5 handles all traffic through a single proxy."
-            ),
-            options=[
-                {"value": "none", "label": "None (Direct Connection)"},
-                {"value": "http", "label": "HTTP/HTTPS Proxy"},
-                {"value": "socks5", "label": "SOCKS5 Proxy"},
-            ],
-            default="none",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-        ),
-        TextField(
-            key="HTTP_PROXY",
-            label="HTTP Proxy",
-            description="HTTP proxy URL (e.g., http://proxy:8080)",
-            placeholder="http://proxy:8080",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-            show_when={"field": "PROXY_MODE", "value": "http"},
-        ),
-        TextField(
-            key="HTTPS_PROXY",
-            label="HTTPS Proxy",
-            description="HTTPS proxy URL (leave empty to use HTTP proxy for HTTPS)",
-            placeholder="http://proxy:8080",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-            show_when={"field": "PROXY_MODE", "value": "http"},
-        ),
-        TextField(
-            key="SOCKS5_PROXY",
-            label="SOCKS5 Proxy",
-            description="SOCKS5 proxy URL. Supports auth: socks5://user:pass@host:port",
-            placeholder="socks5://localhost:1080",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-            show_when={"field": "PROXY_MODE", "value": "socks5"},
-        ),
-    ]
-
-
-@register_settings("downloads", "Downloads", icon="folder", order=5)
-def download_settings():
-    """Configure download behavior and file locations."""
-    return [
-        TextField(
-            key="INGEST_DIR",
-            label="Download Directory",
-            description="Directory where downloaded files are saved.",
-            default="/cwa-book-ingest",
-            required=True,
-        ),
-        CheckboxField(
-            key="USE_BOOK_TITLE",
-            label="Use Book Info as Filename",
-            description="Save files using Author, Title and Year instead of ID. May cause issues with special characters.",
-            default=True,
-        ),
-        CheckboxField(
-            key="AUTO_OPEN_DOWNLOADS_SIDEBAR",
-            label="Auto-Open Downloads Sidebar",
-            description="Automatically open the downloads sidebar when a new download is queued.",
-            default=False,
-            env_supported=False,  # UI-only setting
-        ),
-        CheckboxField(
-            key="DOWNLOAD_TO_BROWSER",
-            label="Download to Browser",
-            description="Automatically download completed files to your browser.",
-            default=False,
-            env_supported=False,  # UI-only setting
-        ),
-        NumberField(
-            key="MAX_CONCURRENT_DOWNLOADS",
-            label="Max Concurrent Downloads",
-            description="Maximum number of simultaneous downloads.",
-            default=3,
-            min_value=1,
-            max_value=10,
-            requires_restart=True,
-        ),
-        NumberField(
-            key="STATUS_TIMEOUT",
-            label="Status Timeout (seconds)",
-            description="How long to keep completed/failed downloads in the queue display.",
-            default=3600,
-            min_value=60,
-            max_value=86400,
-        ),
-        CheckboxField(
-            key="USE_CONTENT_TYPE_DIRECTORIES",
-            label="Configure Content-Type Directories",
-            description="Show options to specify custom directories for each content type (fiction, non-fiction, comics, etc.). If a directory is set, that content type will be saved there instead of the default download directory.",
-            default=False,
-            env_supported=False,  # UI-only toggle to show/hide directory fields
-        ),
-        HeadingField(
-            key="content_type_directories_heading",
-            title="Content-Type Directories",
-            description="Specify custom directories for each content type. Leave empty to use the default download directory.",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_BOOK_FICTION",
-            label="Fiction Books",
-            placeholder="/cwa-book-ingest/fiction",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_BOOK_NON_FICTION",
-            label="Non-Fiction Books",
-            placeholder="/cwa-book-ingest/non-fiction",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_BOOK_UNKNOWN",
-            label="Unknown Books",
-            placeholder="/cwa-book-ingest/unknown",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_MAGAZINE",
-            label="Magazines",
-            placeholder="/cwa-book-ingest/magazines",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_COMIC_BOOK",
-            label="Comic Books",
-            placeholder="/cwa-book-ingest/comics",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_AUDIOBOOK",
-            label="Audiobooks",
-            placeholder="/cwa-book-ingest/audiobooks",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_STANDARDS_DOCUMENT",
-            label="Standards Documents",
-            placeholder="/cwa-book-ingest/standards",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_MUSICAL_SCORE",
-            label="Musical Scores",
-            placeholder="/cwa-book-ingest/scores",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_OTHER",
-            label="Other",
-            placeholder="/cwa-book-ingest/other",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-    ]
-
-
-def _get_source_priority_options():
-    """Build source priority options with dynamic disabled states."""
-    from cwa_book_downloader.core.config import config
-
-    has_donator_key = bool(config.get("AA_DONATOR_KEY", ""))
-    use_cf_bypass = config.get("USE_CF_BYPASS", True)
-    using_external_bypasser = config.get("USING_EXTERNAL_BYPASSER", False)
-    has_internal_bypasser = use_cf_bypass and not using_external_bypasser
-
-    return [
-        {
-            "id": "aa-fast",
-            "label": "Anna's Archive (Fast)",
-            "description": "Fast downloads for donators",
-            "isLocked": not has_donator_key,
-            "disabledReason": "Requires AA Donator Key" if not has_donator_key else None,
-        },
-        {
-            "id": "welib",
-            "label": "Welib",
-            "description": "Alternative mirror with good availability",
-            "isLocked": not has_internal_bypasser,
-            "disabledReason": "Requires internal bypasser" if not has_internal_bypasser else None,
-        },
-        {
-            "id": "aa-slow-nowait",
-            "label": "Anna's Archive (Slowest, No Waitlist)",
-            "description": "Partner servers without countdown",
-        },
-        {
-            "id": "aa-slow-wait",
-            "label": "Anna's Archive (Slow, Waitlist)",
-            "description": "Partner servers with countdown timer",
-        },
-        {
-            "id": "libgen",
-            "label": "Libgen",
-            "description": "Library Genesis mirrors",
-        },
-        {
-            "id": "zlib",
-            "label": "Z-Library",
-            "description": "Z-Library mirrors (requires Cloudflare bypass)",
-            "isLocked": not has_internal_bypasser,
-            "disabledReason": "Requires internal bypasser" if not has_internal_bypasser else None,
-        },
-    ]
-
-
-def _get_default_source_priority():
-    """Default source priority order, respecting legacy env vars.
-
-    ALLOW_USE_WELIB (default true) controls whether welib is enabled.
-    PRIORITIZE_WELIB (default false) controls whether welib is moved to position 1.
-    """
-    from cwa_book_downloader.config.env import _LEGACY_PRIORITIZE_WELIB, _LEGACY_ALLOW_USE_WELIB
-
-    welib_entry = {"id": "welib", "enabled": _LEGACY_ALLOW_USE_WELIB}
-
-    priority = [
-        {"id": "aa-fast", "enabled": True},
-        {"id": "aa-slow-nowait", "enabled": True},
-        {"id": "aa-slow-wait", "enabled": True},
-        {"id": "libgen", "enabled": True},
-    ]
-
-    if _LEGACY_PRIORITIZE_WELIB:
-        priority.insert(1, welib_entry)  # After aa-fast
-    else:
-        priority.append(welib_entry)  # Before zlib
-
-    # Z-Library last - it's quite brittle
-    priority.append({"id": "zlib", "enabled": True})
-
-    return priority
-
-
-@register_settings("download_sources", "Download Sources", icon="download", order=21, group="direct_download")
-def download_source_settings():
-    """Settings for download source behavior."""
-    return [
-        HeadingField(
-            key="source_priority_heading",
-            title="Source Priority",
-            description="Configure which download sources to use and in what order.",
-        ),
-        OrderableListField(
-            key="SOURCE_PRIORITY",
-            label="Download Source Order",
-            description="Drag to reorder. Sources are tried from top to bottom until a download succeeds.",
-            options=_get_source_priority_options,
-            default=_get_default_source_priority(),
-        ),
-        NumberField(
-            key="MAX_RETRY",
-            label="Max Retries",
-            description="Maximum retry attempts for failed downloads.",
-            default=10,
-            min_value=1,
-            max_value=50,
-        ),
-        NumberField(
-            key="DEFAULT_SLEEP",
-            label="Retry Delay (seconds)",
-            description="Wait time between download retry attempts.",
-            default=5,
-            min_value=1,
-            max_value=60,
-        ),
-        HeadingField(
-            key="aa_settings_heading",
-            title="Anna's Archive",
-            description="Configure Anna's Archive mirror and donator settings.",
-        ),
-        SelectField(
-            key="AA_BASE_URL",
-            label="Anna's Archive URL",
-            description="Primary Anna's Archive mirror to use. 'auto' selects automatically.",
-            options=[
-                {"value": "auto", "label": "Auto (Recommended)"},
-                {"value": "https://annas-archive.org", "label": "annas-archive.org"},
-                {"value": "https://annas-archive.se", "label": "annas-archive.se"},
-                {"value": "https://annas-archive.li", "label": "annas-archive.li"},
-            ],
-            default="auto",
-        ),
-        TextField(
-            key="AA_ADDITIONAL_URLS",
-            label="Additional AA Mirrors",
-            description="Comma-separated list of additional Anna's Archive mirror URLs.",
-            placeholder="https://example.com,https://another.com",
-        ),
-        PasswordField(
-            key="AA_DONATOR_KEY",
-            label="Anna's Archive Donator Key",
-            description="Optional donator key for faster downloads from Anna's Archive.",
-        ),
-    ]
-
-
-@register_settings("cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download")
-def cloudflare_bypass_settings():
-    """Settings for Cloudflare bypass behavior."""
-    return [
-        CheckboxField(
-            key="USE_CF_BYPASS",
-            label="Enable Cloudflare Bypass",
-            description="Attempt to bypass Cloudflare protection on download sites.",
-            default=True,
-            requires_restart=True,
-        ),
-        CheckboxField(
-            key="BYPASS_WARMUP_ON_CONNECT",
-            label="Warmup on Connect",
-            description="Pre-warm the bypasser when user connects to Web App UI",
-            default=True,
-        ),
-        NumberField(
-            key="BYPASS_RELEASE_INACTIVE_MIN",
-            label="Release Inactive (minutes)",
-            description="Release bypasser resources after this many minutes of inactivity.",
-            default=5,
-            min_value=1,
-            max_value=60,
-        ),
-        CheckboxField(
-            key="USING_EXTERNAL_BYPASSER",
-            label="Use External Bypasser",
-            description="Use FlareSolverr or similar external service instead of built-in bypasser. Caution: May have limitations with custom DNS, Tor and proxies. You may experience slower downloads and and poorer reliability compared to the internal bypasser.",
-            default=False,
-            requires_restart=True,
-        ),
-        TextField(
-            key="EXT_BYPASSER_URL",
-            label="External Bypasser URL",
-            description="URL of the external bypasser service (e.g., FlareSolverr).",
-            default="http://flaresolverr:8191",
-            placeholder="http://flaresolverr:8191",
-            requires_restart=True,
-            show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
-        ),
-        TextField(
-            key="EXT_BYPASSER_PATH",
-            label="External Bypasser Path",
-            description="API path for the external bypasser.",
-            default="/v1",
-            placeholder="/v1",
-            requires_restart=True,
-            show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
-        ),
-        NumberField(
-            key="EXT_BYPASSER_TIMEOUT",
-            label="External Bypasser Timeout (ms)",
-            description="Timeout for external bypasser requests in milliseconds.",
-            default=60000,
-            min_value=10000,
-            max_value=300000,
-            requires_restart=True,
-            show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
-        ),
-    ]
-
-
-@register_settings("advanced", "Advanced", icon="cog", order=15)
-def advanced_settings():
-    """Advanced settings for power users."""
-    return [
-        TextField(
-            key="CUSTOM_SCRIPT",
-            label="Custom Script Path",
-            description="Path to a script to run after each successful download. Must be executable.",
-            placeholder="/path/to/script.sh",
-        ),
-        CheckboxField(
-            key="DEBUG",
-            label="Debug Mode",
-            description="Enable verbose logging to console and file. Not recommended for normal use.",
-            default=False,
-            requires_restart=True,
-        ),
-        NumberField(
-            key="MAIN_LOOP_SLEEP_TIME",
-            label="Queue Check Interval (seconds)",
-            description="How often the download queue is checked for new items.",
-            default=5,
-            min_value=1,
-            max_value=60,
-            requires_restart=True,
-        ),
-        NumberField(
-            key="DOWNLOAD_PROGRESS_UPDATE_INTERVAL",
-            label="Progress Update Interval (seconds)",
-            description="How often download progress is broadcast to the UI.",
-            default=1,
-            min_value=1,
-            max_value=10,
-            requires_restart=True,
-        ),
-        HeadingField(
-            key="covers_cache_heading",
-            title="Cover Image Cache",
-            description="Cache book cover images locally for faster loading. Works for both Direct Download and Universal mode.",
-        ),
-        CheckboxField(
-            key="COVERS_CACHE_ENABLED",
-            label="Enable Cover Cache",
-            description="Cache book covers on the server for faster loading.",
-            default=True,
-        ),
-        NumberField(
-            key="COVERS_CACHE_TTL",
-            label="Cache TTL (days)",
-            description="How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork).",
-            default=0,
-            min_value=0,
-            max_value=365,
-        ),
-        NumberField(
-            key="COVERS_CACHE_MAX_SIZE_MB",
-            label="Max Cache Size (MB)",
-            description="Maximum disk space for cached covers. Oldest images are removed when limit is reached.",
-            default=500,
-            min_value=50,
-            max_value=5000,
-        ),
-        ActionButton(
-            key="clear_covers_cache",
-            label="Clear Cover Cache",
-            description="Delete all cached cover images.",
-            style="danger",
-            callback=_clear_covers_cache,
-        ),
-        HeadingField(
-            key="metadata_cache_heading",
-            title="Metadata Cache",
-            description="Cache book metadata from providers (Hardcover, Open Library) to reduce API calls and speed up repeated searches.",
-        ),
-        CheckboxField(
-            key="METADATA_CACHE_ENABLED",
-            label="Enable Metadata Caching",
-            description="When disabled, all metadata searches hit the provider API directly.",
-            default=True,
-        ),
-        NumberField(
-            key="METADATA_CACHE_SEARCH_TTL",
-            label="Search Results Cache (seconds)",
-            description="How long to cache search results. Default: 300 (5 minutes). Max: 604800 (7 days).",
-            default=300,
-            min_value=60,
-            max_value=604800,
-            show_when={"field": "METADATA_CACHE_ENABLED", "value": True},
-        ),
-        NumberField(
-            key="METADATA_CACHE_BOOK_TTL",
-            label="Book Details Cache (seconds)",
-            description="How long to cache individual book details. Default: 600 (10 minutes). Max: 604800 (7 days).",
-            default=600,
-            min_value=60,
-            max_value=604800,
-            show_when={"field": "METADATA_CACHE_ENABLED", "value": True},
-        ),
-        ActionButton(
-            key="clear_metadata_cache",
-            label="Clear Metadata Cache",
-            description="Clear all cached search results and book details.",
-            style="danger",
-            callback=_clear_metadata_cache,
-        ),
-    ]

cwa_book_downloader/core/__init__.py

@@ -1,5 +0,0 @@
-"""Core module - shared models, queue, and utilities."""
-
-from cwa_book_downloader.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
-from cwa_book_downloader.core.queue import BookQueue, book_queue
-from cwa_book_downloader.core.logger import setup_logger

cwa_book_downloader/download/archive.py

@@ -1,354 +0,0 @@
-"""Archive extraction utilities for downloaded book archives."""
-
-import os
-import shutil
-import zipfile
-from dataclasses import dataclass
-from pathlib import Path
-from typing import List, Optional, Tuple
-
-from cwa_book_downloader.core.logger import setup_logger
-
-logger = setup_logger(__name__)
-
-# Check for rarfile availability at module load
-try:
-    import rarfile
-
-    RAR_AVAILABLE = True
-except ImportError:
-    RAR_AVAILABLE = False
-    logger.warning("rarfile not installed - RAR extraction disabled")
-
-# Book file extensions that should be kept after extraction
-BOOK_EXTENSIONS = frozenset({
-    "epub", "mobi", "azw", "azw3", "pdf", "fb2", "djvu",
-    "cbz", "cbr", "txt", "rtf", "doc", "docx", "lit", "pdb",
-})
-
-
-class ArchiveExtractionError(Exception):
-    """Raised when archive extraction fails."""
-
-    pass
-
-
-class PasswordProtectedError(ArchiveExtractionError):
-    """Raised when archive requires a password."""
-
-    pass
-
-
-class CorruptedArchiveError(ArchiveExtractionError):
-    """Raised when archive is corrupted."""
-
-    pass
-
-
-def is_archive(file_path: Path) -> bool:
-    """Check if file is a supported archive format."""
-    suffix = file_path.suffix.lower().lstrip(".")
-    return suffix in ("zip", "rar")
-
-
-def _is_book_file(file_path: Path) -> bool:
-    """Check if file is a recognized book format."""
-    ext = file_path.suffix.lower().lstrip(".")
-    return ext in BOOK_EXTENSIONS
-
-
-def _filter_book_files(extracted_files: List[Path]) -> Tuple[List[Path], List[Path]]:
-    """
-    Filter extracted files to only book formats.
-
-    Returns:
-        Tuple of (book_files, non_book_files)
-    """
-    book_files = []
-    non_book_files = []
-
-    for file_path in extracted_files:
-        if _is_book_file(file_path):
-            book_files.append(file_path)
-        else:
-            non_book_files.append(file_path)
-
-    return book_files, non_book_files
-
-
-def extract_archive(
-    archive_path: Path,
-    output_dir: Path,
-) -> Tuple[List[Path], List[str]]:
-    """
-    Extract book files from an archive.
-
-    Extracts all files, then filters to only keep recognized book formats.
-    Non-book files (HTML, images, etc.) are deleted.
-
-    Args:
-        archive_path: Path to the archive file
-        output_dir: Directory to extract files to
-
-    Returns:
-        Tuple of (extracted_book_file_paths, warnings)
-
-    Raises:
-        ArchiveExtractionError: If extraction fails
-        PasswordProtectedError: If archive requires password
-        CorruptedArchiveError: If archive is corrupted
-    """
-    suffix = archive_path.suffix.lower().lstrip(".")
-
-    if suffix == "zip":
-        extracted_files, warnings = _extract_zip(archive_path, output_dir)
-    elif suffix == "rar":
-        extracted_files, warnings = _extract_rar(archive_path, output_dir)
-    else:
-        raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
-
-    # Filter to only book files, delete non-book files
-    book_files, non_book_files = _filter_book_files(extracted_files)
-
-    for non_book_file in non_book_files:
-        try:
-            non_book_file.unlink()
-            logger.debug(f"Deleted non-book file: {non_book_file.name}")
-        except OSError as e:
-            logger.warning(f"Failed to delete non-book file {non_book_file}: {e}")
-
-    if non_book_files:
-        warnings.append(f"Skipped {len(non_book_files)} non-book file(s)")
-
-    return book_files, warnings
-
-
-def _extract_zip(
-    archive_path: Path,
-    output_dir: Path,
-) -> Tuple[List[Path], List[str]]:
-    """Extract files from a ZIP archive."""
-    extracted_files = []
-    warnings = []
-
-    try:
-        with zipfile.ZipFile(archive_path, "r") as zf:
-            # Check for password protection
-            for info in zf.infolist():
-                if info.flag_bits & 0x1:  # Encrypted flag
-                    raise PasswordProtectedError("ZIP archive is password protected")
-
-            # Test archive integrity
-            bad_file = zf.testzip()
-            if bad_file:
-                raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
-
-            # Extract all files
-            for info in zf.infolist():
-                if info.is_dir():
-                    continue
-
-                # Use only filename, strip directory path (security: prevent path traversal)
-                filename = Path(info.filename).name
-                if not filename:
-                    continue
-
-                # Extract to output_dir with flat structure
-                target_path = output_dir / filename
-                target_path = _handle_duplicate_filename(target_path)
-
-                with zf.open(info) as src, open(target_path, "wb") as dst:
-                    dst.write(src.read())
-
-                extracted_files.append(target_path)
-                logger.debug(f"Extracted: {filename}")
-
-    except zipfile.BadZipFile as e:
-        raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
-    except PermissionError as e:
-        raise ArchiveExtractionError(f"Permission denied: {e}")
-
-    return extracted_files, warnings
-
-
-def _extract_rar(
-    archive_path: Path,
-    output_dir: Path,
-) -> Tuple[List[Path], List[str]]:
-    """Extract files from a RAR archive."""
-    if not RAR_AVAILABLE:
-        raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
-
-    extracted_files = []
-    warnings = []
-
-    try:
-        with rarfile.RarFile(archive_path, "r") as rf:
-            # Check for password protection
-            if rf.needs_password():
-                raise PasswordProtectedError("RAR archive is password protected")
-
-            # Test archive integrity
-            rf.testrar()
-
-            # Extract all files
-            for info in rf.infolist():
-                if info.is_dir():
-                    continue
-
-                # Use only filename, strip directory path (security: prevent path traversal)
-                filename = Path(info.filename).name
-                if not filename:
-                    continue
-
-                # Extract to output_dir with flat structure
-                target_path = output_dir / filename
-                target_path = _handle_duplicate_filename(target_path)
-
-                with rf.open(info) as src, open(target_path, "wb") as dst:
-                    dst.write(src.read())
-
-                extracted_files.append(target_path)
-                logger.debug(f"Extracted: {filename}")
-
-    except rarfile.BadRarFile as e:
-        raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
-    except rarfile.RarCannotExec:
-        raise ArchiveExtractionError("unrar binary not found - install unrar package")
-    except PermissionError as e:
-        raise ArchiveExtractionError(f"Permission denied: {e}")
-
-    return extracted_files, warnings
-
-
-def _handle_duplicate_filename(target_path: Path) -> Path:
-    """Handle duplicate filenames by appending counter."""
-    if not target_path.exists():
-        return target_path
-
-    base = target_path.stem
-    ext = target_path.suffix
-    parent = target_path.parent
-    counter = 1
-
-    while target_path.exists():
-        target_path = parent / f"{base}_{counter}{ext}"
-        counter += 1
-
-    return target_path
-
-
-@dataclass
-class ArchiveResult:
-    """Result of archive processing."""
-
-    success: bool
-    final_paths: List[Path]
-    message: str
-    error: Optional[str] = None
-
-
-def process_archive(
-    archive_path: Path,
-    temp_dir: Path,
-    ingest_dir: Path,
-    archive_id: str,
-) -> ArchiveResult:
-    """
-    Process an archive file: extract, filter to book files, move to ingest.
-
-    This is the main entry point for archive handling, usable by any download handler.
-
-    Args:
-        archive_path: Path to the downloaded archive file
-        temp_dir: Base temp directory for extraction (e.g., TMP_DIR)
-        ingest_dir: Final destination directory for book files
-        archive_id: Unique identifier for temp directory naming
-
-    Returns:
-        ArchiveResult with success status, final paths, and status message
-    """
-    extract_dir = temp_dir / f"extract_{archive_id}"
-
-    try:
-        # Create temp extraction directory
-        os.makedirs(extract_dir, exist_ok=True)
-        os.makedirs(ingest_dir, exist_ok=True)
-
-        # Extract to temp directory (filters to book files only)
-        extracted_files, warnings = extract_archive(archive_path, extract_dir)
-
-        if not extracted_files:
-            # Clean up and return error
-            shutil.rmtree(extract_dir, ignore_errors=True)
-            archive_path.unlink(missing_ok=True)
-            return ArchiveResult(
-                success=False,
-                final_paths=[],
-                message="",
-                error="No book files found in archive",
-            )
-
-        for warning in warnings:
-            logger.debug(warning)
-
-        logger.info(f"Extracted {len(extracted_files)} book file(s) from archive")
-
-        # Move book files to ingest folder
-        final_paths = []
-        for extracted_file in extracted_files:
-            final_path = ingest_dir / extracted_file.name
-            final_path = _handle_duplicate_filename(final_path)
-            shutil.move(str(extracted_file), str(final_path))
-            final_paths.append(final_path)
-            logger.debug(f"Moved to ingest: {final_path.name}")
-
-        # Clean up temp extraction directory and archive
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-
-        # Build success message with extracted formats
-        formats = [p.suffix.lstrip(".").upper() for p in final_paths]
-        if len(formats) == 1:
-            message = f"Extracted: {formats[0]}"
-        else:
-            message = f"Extracted: {len(formats)} files ({', '.join(formats)})"
-
-        return ArchiveResult(
-            success=True,
-            final_paths=final_paths,
-            message=message,
-        )
-
-    except PasswordProtectedError:
-        logger.error(f"Password-protected archive: {archive_path.name}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-        return ArchiveResult(
-            success=False,
-            final_paths=[],
-            message="",
-            error="Archive is password protected",
-        )
-
-    except CorruptedArchiveError as e:
-        logger.error(f"Corrupted archive: {e}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-        return ArchiveResult(
-            success=False,
-            final_paths=[],
-            message="",
-            error=f"Corrupted archive: {e}",
-        )
-
-    except ArchiveExtractionError as e:
-        logger.error(f"Archive extraction failed: {e}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-        return ArchiveResult(
-            success=False,
-            final_paths=[],
-            message="",
-            error=f"Extraction failed: {e}",
-        )

cwa_book_downloader/download_clients/__init__.py

@@ -1,55 +0,0 @@
-"""External download client integrations (qBittorrent, SABnzbd, etc.)."""
-
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
-from enum import Enum
-
-
-class DownloadStatus(Enum):
-    """Status of a download in an external client."""
-    QUEUED = "queued"
-    DOWNLOADING = "downloading"
-    PAUSED = "paused"
-    COMPLETED = "completed"
-    FAILED = "failed"
-    SEEDING = "seeding"  # Torrents only
-
-
-@dataclass
-class ClientDownloadProgress:
-    """Progress info from external download client."""
-    status: DownloadStatus
-    progress: float                  # 0-100
-    download_speed: Optional[int]    # bytes/sec
-    eta: Optional[int]               # seconds remaining
-    save_path: Optional[str]         # Where the file will be/is
-
-
-class DownloadClient(ABC):
-    """Abstract base class for download clients."""
-
-    @abstractmethod
-    def add_download(self, url: str, title: str) -> str:
-        """Add a download (torrent/magnet or NZB URL). Returns download ID for tracking."""
-        pass
-
-    @abstractmethod
-    def get_download(self, download_id: str) -> Optional[ClientDownloadProgress]:
-        """Get progress of a specific download."""
-        pass
-
-    @abstractmethod
-    def list_downloads(self) -> List[Tuple[str, ClientDownloadProgress]]:
-        """List all downloads with their progress."""
-        pass
-
-    @abstractmethod
-    def get_completed_path(self, download_id: str) -> Optional[str]:
-        """Get the path to completed download."""
-        pass
-
-    @abstractmethod
-    def test_connection(self) -> bool:
-        """Test if the client is reachable and credentials are valid."""
-        pass

cwa_book_downloader/metadata_providers/hardcover.py

@@ -1,752 +0,0 @@
-"""Hardcover.app metadata provider. Requires API key."""
-
-import requests
-from typing import Any, Dict, List, Optional
-
-from cwa_book_downloader.core.cache import cacheable
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.settings_registry import (
-    register_settings,
-    CheckboxField,
-    PasswordField,
-    SelectField,
-    ActionButton,
-    HeadingField,
-)
-from cwa_book_downloader.core.config import config as app_config
-from cwa_book_downloader.metadata_providers import (
-    BookMetadata,
-    DisplayField,
-    MetadataProvider,
-    MetadataSearchOptions,
-    SearchType,
-    SortOrder,
-    register_provider,
-    register_provider_kwargs,
-    TextSearchField,
-)
-
-logger = setup_logger(__name__)
-
-HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql"
-
-
-# Mapping from abstract sort order to Hardcover sort parameter
-# Note: release_year is more consistently populated than release_date_i
-SORT_MAPPING: Dict[SortOrder, str] = {
-    SortOrder.RELEVANCE: "_text_match:desc,users_count:desc",
-    SortOrder.POPULARITY: "users_count:desc",
-    SortOrder.RATING: "rating:desc",
-    SortOrder.NEWEST: "release_year:desc",
-    SortOrder.OLDEST: "release_year:asc",
-}
-
-# Mapping from abstract search type to Hardcover fields parameter
-SEARCH_TYPE_FIELDS: Dict[SearchType, str] = {
-    SearchType.GENERAL: "title,isbns,series_names,author_names,alternative_titles",
-    SearchType.TITLE: "title,alternative_titles",
-    SearchType.AUTHOR: "author_names",
-    # ISBN is handled separately via search_by_isbn()
-}
-
-
-def _combine_headline_description(headline: Optional[str], description: Optional[str]) -> Optional[str]:
-    """Combine headline (tagline) and description into a single description.
-
-    Hardcover stores a short 'headline' (tagline/promotional text) separately
-    from the main description. This combines them for display.
-
-    Args:
-        headline: Short promotional text or tagline.
-        description: Full book synopsis/description.
-
-    Returns:
-        Combined description with headline as the first line, or just one if only one exists.
-    """
-    if headline and description:
-        # Add headline as first paragraph, followed by description
-        return f"{headline}\n\n{description}"
-    elif headline:
-        return headline
-    elif description:
-        return description
-    return None
-
-
-@register_provider_kwargs("hardcover")
-def _hardcover_kwargs() -> Dict[str, Any]:
-    """Provide Hardcover-specific constructor kwargs."""
-    return {"api_key": app_config.get("HARDCOVER_API_KEY", "")}
-
-
-@register_provider("hardcover")
-class HardcoverProvider(MetadataProvider):
-    """Hardcover.app metadata provider using GraphQL API."""
-
-    name = "hardcover"
-    display_name = "Hardcover"
-    requires_auth = True
-    supported_sorts = [
-        SortOrder.RELEVANCE,
-        SortOrder.POPULARITY,
-        SortOrder.RATING,
-        SortOrder.NEWEST,
-        SortOrder.OLDEST,
-    ]
-    search_fields = [
-        TextSearchField(
-            key="author",
-            label="Author",
-            description="Search by author name",
-        ),
-        TextSearchField(
-            key="title",
-            label="Title",
-            description="Search by book title",
-        ),
-    ]
-
-    def __init__(self, api_key: Optional[str] = None):
-        """Initialize provider with API key.
-
-        Args:
-            api_key: Hardcover API key. If not provided, uses config singleton.
-        """
-        self.api_key = api_key or app_config.get("HARDCOVER_API_KEY", "")
-        self.session = requests.Session()
-        if self.api_key:
-            self.session.headers.update({
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json",
-            })
-
-    def is_available(self) -> bool:
-        """Check if provider is configured with an API key."""
-        return bool(self.api_key)
-
-    def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
-        """Search for books using Hardcover's search API.
-
-        Args:
-            options: Search options (query, type, sort, pagination, fields).
-
-        Returns:
-            List of BookMetadata objects.
-        """
-        if not self.api_key:
-            logger.warning("Hardcover API key not configured")
-            return []
-
-        # Handle ISBN search separately
-        if options.search_type == SearchType.ISBN:
-            result = self.search_by_isbn(options.query)
-            return [result] if result else []
-
-        # Build cache key from options (include fields for cache differentiation)
-        fields_key = ":".join(f"{k}={v}" for k, v in sorted(options.fields.items()))
-        cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.limit}:{options.page}:{fields_key}"
-        return self._search_cached(cache_key, options)
-
-    @cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="hardcover:search")
-    def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]:
-        """Cached search implementation.
-
-        Args:
-            cache_key: Cache key (used by decorator).
-            options: Search options.
-
-        Returns:
-            List of BookMetadata objects.
-        """
-        # Determine query and fields based on custom search fields
-        # Field-first search: when a specific field has a value, search that field
-        author_value = options.fields.get("author", "").strip()
-        title_value = options.fields.get("title", "").strip()
-
-        logger.debug(f"Field-first search check: author_value='{author_value}', title_value='{title_value}'")
-
-        # Determine what to search and which fields to target
-        # Note: Hardcover API requires 'weights' when using 'fields' parameter
-        if author_value and not title_value:
-            # Author-only search: search author_names field with author query
-            query = author_value
-            search_fields = "author_names"
-            search_weights = "1"
-            logger.debug(f"Author-only search: query='{query}', fields='{search_fields}'")
-        elif title_value and not author_value:
-            # Title-only search: search title fields with title query
-            query = title_value
-            search_fields = "title,alternative_titles"
-            search_weights = "5,1"
-            logger.debug(f"Title-only search: query='{query}', fields='{search_fields}'")
-        elif author_value and title_value:
-            # Both provided: combine into query, search both fields
-            query = f"{title_value} {author_value}"
-            search_fields = "title,alternative_titles,author_names"
-            search_weights = "5,1,3"
-            logger.debug(f"Combined search: query='{query}', fields='{search_fields}'")
-        else:
-            # No custom fields: use general query with all default fields
-            query = options.query
-            search_fields = None
-            search_weights = None
-            logger.debug(f"General search: query='{query}', no field restriction")
-
-        # Build GraphQL query with optional fields/weights parameters
-        if search_fields:
-            graphql_query = """
-            query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) {
-                search(
-                    query: $query,
-                    query_type: "Book",
-                    per_page: $limit,
-                    page: $page,
-                    sort: $sort,
-                    fields: $fields,
-                    weights: $weights
-                ) {
-                    results
-                }
-            }
-            """
-        else:
-            graphql_query = """
-            query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) {
-                search(
-                    query: $query,
-                    query_type: "Book",
-                    per_page: $limit,
-                    page: $page,
-                    sort: $sort
-                ) {
-                    results
-                }
-            }
-            """
-
-        # Map abstract sort order to Hardcover's sort parameter
-        sort_param = SORT_MAPPING.get(options.sort, SORT_MAPPING[SortOrder.RELEVANCE])
-
-        variables = {
-            "query": query,
-            "limit": options.limit,
-            "page": options.page,
-            "sort": sort_param,
-        }
-
-        if search_fields:
-            variables["fields"] = search_fields
-            variables["weights"] = search_weights
-
-        logger.debug(f"GraphQL variables: {variables}")
-
-        try:
-            result = self._execute_query(graphql_query, variables)
-            if not result:
-                logger.debug("Hardcover search: No result from API")
-                return []
-
-            search_data = result.get("search", {})
-
-            # Results is a Typesense response object with hits array
-            results_obj = search_data.get("results", {})
-            if isinstance(results_obj, dict):
-                hits = results_obj.get("hits", [])
-            else:
-                hits = results_obj if isinstance(results_obj, list) else []
-
-            # Parse the search results - each hit has a 'document' field
-            books = []
-            for hit in hits:
-                # Get the document from the hit
-                item = hit.get("document", hit) if isinstance(hit, dict) else hit
-                if isinstance(item, dict):
-                    book = self._parse_search_result(item)
-                    if book:
-                        books.append(book)
-
-            logger.info(f"Hardcover search '{query}' (fields={search_fields}) returned {len(books)} results")
-            return books
-
-        except Exception as e:
-            logger.error(f"Hardcover search error: {e}")
-            return []
-
-    @cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="hardcover:book")
-    def get_book(self, book_id: str) -> Optional[BookMetadata]:
-        """Get book details by Hardcover ID.
-
-        Args:
-            book_id: Hardcover book ID.
-
-        Returns:
-            BookMetadata or None if not found.
-        """
-        if not self.api_key:
-            logger.warning("Hardcover API key not configured")
-            return None
-
-        # Query for specific book by ID
-        # Note: API has max depth of 3, so use cached_* fields instead of nested relationships
-        graphql_query = """
-        query GetBook($id: Int!) {
-            books(where: {id: {_eq: $id}}, limit: 1) {
-                id
-                title
-                slug
-                release_date
-                headline
-                description
-                pages
-                cached_image
-                cached_contributors
-                cached_tags
-                default_physical_edition {
-                    isbn_10
-                    isbn_13
-                }
-            }
-        }
-        """
-
-        try:
-            book_id_int = int(book_id)
-            result = self._execute_query(graphql_query, {"id": book_id_int})
-            if not result:
-                return None
-
-            books = result.get("books", [])
-            if not books:
-                return None
-
-            return self._parse_book(books[0])
-
-        except ValueError:
-            logger.error(f"Invalid book ID: {book_id}")
-            return None
-        except Exception as e:
-            logger.error(f"Hardcover get_book error: {e}")
-            return None
-
-    @cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="hardcover:isbn")
-    def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
-        """Search for a book by ISBN.
-
-        Args:
-            isbn: ISBN-10 or ISBN-13.
-
-        Returns:
-            BookMetadata or None if not found.
-        """
-        if not self.api_key:
-            logger.warning("Hardcover API key not configured")
-            return None
-
-        # Clean ISBN (remove hyphens)
-        clean_isbn = isbn.replace("-", "").strip()
-
-        # Search for editions with matching ISBN
-        # Note: API has max depth of 3, so use cached_* fields instead of nested relationships
-        graphql_query = """
-        query SearchByISBN($isbn: String!) {
-            editions(
-                where: {
-                    _or: [
-                        {isbn_10: {_eq: $isbn}},
-                        {isbn_13: {_eq: $isbn}}
-                    ]
-                },
-                limit: 1
-            ) {
-                isbn_10
-                isbn_13
-                book {
-                    id
-                    title
-                    slug
-                    release_date
-                    headline
-                    description
-                    pages
-                    cached_image
-                    cached_contributors
-                    cached_tags
-                }
-            }
-        }
-        """
-
-        try:
-            result = self._execute_query(graphql_query, {"isbn": clean_isbn})
-            if not result:
-                return None
-
-            editions = result.get("editions", [])
-            if not editions:
-                logger.debug(f"No Hardcover book found for ISBN: {isbn}")
-                return None
-
-            edition = editions[0]
-            book_data = edition.get("book", {})
-            if not book_data:
-                return None
-
-            # Add ISBN data from edition to book data
-            book_data["isbn_10"] = edition.get("isbn_10")
-            book_data["isbn_13"] = edition.get("isbn_13")
-
-            return self._parse_book(book_data)
-
-        except Exception as e:
-            logger.error(f"Hardcover ISBN search error: {e}")
-            return None
-
-    def _execute_query(self, query: str, variables: Dict[str, Any]) -> Optional[Dict]:
-        """Execute a GraphQL query.
-
-        Args:
-            query: GraphQL query string.
-            variables: Query variables.
-
-        Returns:
-            Response data dict or None on error.
-        """
-        try:
-            response = self.session.post(
-                HARDCOVER_API_URL,
-                json={"query": query, "variables": variables},
-                timeout=15
-            )
-            response.raise_for_status()
-
-            data = response.json()
-
-            if "errors" in data:
-                logger.error(f"GraphQL errors: {data['errors']}")
-                return None
-
-            return data.get("data")
-
-        except requests.Timeout:
-            logger.warning("Hardcover API request timed out")
-            return None
-        except requests.HTTPError as e:
-            if e.response.status_code == 401:
-                logger.error("Hardcover API key is invalid")
-            else:
-                logger.error(f"Hardcover API HTTP error: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"Hardcover API request failed: {e}")
-            return None
-
-    def _parse_search_result(self, item: Dict) -> Optional[BookMetadata]:
-        """Parse a search result item into BookMetadata.
-
-        Args:
-            item: Search result item dict.
-
-        Returns:
-            BookMetadata or None if parsing fails.
-        """
-        try:
-            book_id = item.get("id") or item.get("document", {}).get("id")
-            title = item.get("title") or item.get("document", {}).get("title")
-
-            if not book_id or not title:
-                return None
-
-            # Extract authors from various possible fields
-            authors = []
-            if "author_names" in item:
-                authors = item["author_names"] if isinstance(item["author_names"], list) else [item["author_names"]]
-            elif "cached_contributors" in item:
-                for contrib in item.get("cached_contributors", []):
-                    if isinstance(contrib, dict) and contrib.get("name"):
-                        authors.append(contrib["name"])
-                    elif isinstance(contrib, str):
-                        authors.append(contrib)
-
-            # Get cover URL
-            cover_url = None
-            if "image" in item and item["image"]:
-                cover_url = item["image"] if isinstance(item["image"], str) else item["image"].get("url")
-
-            # Extract year - prefer release_year if available, fall back to release_date
-            publish_year = None
-            if "release_year" in item and item["release_year"]:
-                try:
-                    publish_year = int(item["release_year"])
-                except (ValueError, TypeError):
-                    pass
-            elif "release_date" in item and item["release_date"]:
-                try:
-                    publish_year = int(str(item["release_date"])[:4])
-                except (ValueError, TypeError):
-                    pass
-
-            slug = item.get("slug", "")
-            source_url = f"https://hardcover.app/books/{slug}" if slug else None
-
-            # Build display fields from Hardcover-specific data
-            display_fields = []
-
-            # Rating (e.g., "4.5 (3,764)")
-            rating = item.get("rating")
-            ratings_count = item.get("ratings_count")
-            if rating is not None:
-                rating_str = f"{rating:.1f}"
-                if ratings_count:
-                    rating_str += f" ({ratings_count:,})"
-                display_fields.append(DisplayField(label="Rating", value=rating_str, icon="star"))
-
-            # Readers (users who have this book)
-            users_count = item.get("users_count")
-            if users_count:
-                display_fields.append(DisplayField(label="Readers", value=f"{users_count:,}", icon="users"))
-
-            # Combine headline and description if both present
-            headline = item.get("headline")
-            description = item.get("description")
-            full_description = _combine_headline_description(headline, description)
-
-            return BookMetadata(
-                provider="hardcover",
-                provider_id=str(book_id),
-                title=title,
-                provider_display_name="Hardcover",
-                authors=authors,
-                cover_url=cover_url,
-                description=full_description,
-                publish_year=publish_year,
-                source_url=source_url,
-                display_fields=display_fields,
-            )
-
-        except Exception as e:
-            logger.debug(f"Failed to parse Hardcover search result: {e}")
-            return None
-
-    def _parse_book(self, book: Dict) -> BookMetadata:
-        """Parse a book object into BookMetadata.
-
-        Args:
-            book: Book data dict from GraphQL response.
-
-        Returns:
-            BookMetadata object.
-        """
-        # Extract authors from cached_contributors (json array) or contributions relationship
-        authors = []
-        if book.get("cached_contributors"):
-            for contrib in book["cached_contributors"]:
-                if isinstance(contrib, dict) and contrib.get("name"):
-                    authors.append(contrib["name"])
-                elif isinstance(contrib, str):
-                    authors.append(contrib)
-        elif book.get("contributions"):
-            # Fallback for contributions relationship (if used)
-            for contrib in book["contributions"]:
-                author = contrib.get("author", {})
-                if author and author.get("name"):
-                    authors.append(author["name"])
-
-        # Get cover URL from cached_image (jsonb) or image relationship
-        cover_url = None
-        if book.get("cached_image"):
-            cached = book["cached_image"]
-            if isinstance(cached, dict):
-                cover_url = cached.get("url")
-            elif isinstance(cached, str):
-                cover_url = cached
-        elif book.get("image"):
-            img = book["image"]
-            cover_url = img if isinstance(img, str) else img.get("url")
-
-        # Extract year from release_date
-        publish_year = None
-        if book.get("release_date"):
-            try:
-                publish_year = int(str(book["release_date"])[:4])
-            except (ValueError, TypeError):
-                pass
-
-        # Extract genres from cached_tags
-        genres = []
-        for tag in book.get("cached_tags", []):
-            if isinstance(tag, dict) and tag.get("tag"):
-                genres.append(tag["tag"])
-            elif isinstance(tag, str):
-                genres.append(tag)
-
-        # Get ISBN from direct fields, default_physical_edition, or editions
-        isbn_10 = book.get("isbn_10")
-        isbn_13 = book.get("isbn_13")
-
-        if not isbn_10 and not isbn_13:
-            # Try default_physical_edition first
-            edition = book.get("default_physical_edition")
-            if edition:
-                isbn_10 = edition.get("isbn_10")
-                isbn_13 = edition.get("isbn_13")
-
-            # Fallback to editions array
-            if not isbn_10 and not isbn_13 and book.get("editions"):
-                for ed in book["editions"]:
-                    if not isbn_10 and ed.get("isbn_10"):
-                        isbn_10 = ed["isbn_10"]
-                    if not isbn_13 and ed.get("isbn_13"):
-                        isbn_13 = ed["isbn_13"]
-                    if isbn_10 and isbn_13:
-                        break
-
-        slug = book.get("slug", "")
-        source_url = f"https://hardcover.app/books/{slug}" if slug else None
-
-        # Combine headline and description if both present
-        headline = book.get("headline")
-        description = book.get("description")
-        full_description = _combine_headline_description(headline, description)
-
-        return BookMetadata(
-            provider="hardcover",
-            provider_id=str(book["id"]),
-            title=book["title"],
-            provider_display_name="Hardcover",
-            authors=authors,
-            isbn_10=isbn_10,
-            isbn_13=isbn_13,
-            cover_url=cover_url,
-            description=full_description,
-            publish_year=publish_year,
-            genres=genres,
-            source_url=source_url,
-        )
-
-
-def _test_hardcover_connection() -> Dict[str, Any]:
-    """Test the Hardcover API connection."""
-    from cwa_book_downloader.core.config import config as app_config
-    from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file
-    from cwa_book_downloader.metadata_providers import get_provider_kwargs
-
-    # Refresh config to pick up any recently saved settings
-    app_config.refresh()
-
-    kwargs = get_provider_kwargs("hardcover")
-    api_key = kwargs.get("api_key")
-
-    # Debug: log key info
-    key_len = len(api_key) if api_key else 0
-    key_preview = f"{api_key[:10]}...{api_key[-10:]}" if key_len > 20 else "(too short)"
-    logger.info(f"Hardcover test: key length={key_len}, preview={key_preview}")
-
-    if not api_key:
-        # Clear any stored username since there's no key
-        _save_connected_username(None)
-        return {"success": False, "message": "No API key configured. Save your key and try again."}
-
-    if key_len < 100:
-        return {"success": False, "message": f"API key seems too short ({key_len} chars). Expected 500+ chars."}
-
-    try:
-        provider = HardcoverProvider(api_key=api_key)
-        # Use the 'me' query to test connection (recommended by API docs)
-        result = provider._execute_query(
-            "query { me { id, username } }",
-            {}
-        )
-        if result is not None:
-            # Handle both single object and array response formats
-            me_data = result.get("me", {})
-            if isinstance(me_data, list) and me_data:
-                me_data = me_data[0]
-            username = me_data.get("username", "Unknown") if isinstance(me_data, dict) else "Unknown"
-
-            # Save the username for persistent display
-            _save_connected_username(username)
-
-            return {"success": True, "message": f"Connected as: {username}"}
-        else:
-            _save_connected_username(None)
-            return {"success": False, "message": "API request failed - check your API key"}
-    except Exception as e:
-        logger.exception("Hardcover connection test failed")
-        _save_connected_username(None)
-        return {"success": False, "message": f"Connection failed: {str(e)}"}
-
-
-def _save_connected_username(username: Optional[str]) -> None:
-    """Save or clear the connected username in config."""
-    from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file
-
-    config = load_config_file("hardcover")
-    if username:
-        config["_connected_username"] = username
-    else:
-        config.pop("_connected_username", None)
-    save_config_file("hardcover", config)
-
-
-def _get_connected_username() -> Optional[str]:
-    """Get the stored connected username."""
-    from cwa_book_downloader.core.settings_registry import load_config_file
-
-    config = load_config_file("hardcover")
-    return config.get("_connected_username")
-
-
-# Hardcover sort options for settings UI
-_HARDCOVER_SORT_OPTIONS = [
-    {"value": "relevance", "label": "Most relevant"},
-    {"value": "popularity", "label": "Most popular"},
-    {"value": "rating", "label": "Highest rated"},
-    {"value": "newest", "label": "Newest"},
-    {"value": "oldest", "label": "Oldest"},
-]
-
-
-@register_settings("hardcover", "Hardcover", icon="book", order=51, group="metadata_providers")
-def hardcover_settings():
-    """Hardcover metadata provider settings."""
-    # Check for connected username to show status
-    connected_user = _get_connected_username()
-    test_button_description = f"Connected as: {connected_user}" if connected_user else "Verify your API key works"
-
-    return [
-        HeadingField(
-            key="hardcover_heading",
-            title="Hardcover",
-            description="A modern book tracking and discovery platform with a comprehensive API.",
-            link_url="https://hardcover.app",
-            link_text="hardcover.app",
-        ),
-        CheckboxField(
-            key="HARDCOVER_ENABLED",
-            label="Enable Hardcover",
-            description="Enable Hardcover as a metadata provider for book searches",
-            default=False,
-        ),
-        PasswordField(
-            key="HARDCOVER_API_KEY",
-            label="API Key",
-            description="Get your API key from hardcover.app/account/api",
-            required=True,
-            env_supported=False,  # UI-only setting, no ENV var support
-        ),
-        ActionButton(
-            key="test_connection",
-            label="Test Connection",
-            description=test_button_description,
-            style="primary",
-            callback=_test_hardcover_connection,
-        ),
-        SelectField(
-            key="HARDCOVER_DEFAULT_SORT",
-            label="Default Sort Order",
-            description="Default sort order for Hardcover search results.",
-            options=_HARDCOVER_SORT_OPTIONS,
-            default="relevance",
-            env_supported=False,  # UI-only setting
-        ),
-    ]

docker-compose.dev.extbp.yml

@@ -0,0 +1,25 @@
+# Local development - External bypasser variant (lite)
+services:
+  shelfmark-lite-dev:
+    extends:
+      file: ./compose/docker-compose.lite.yml
+      service: shelfmark-lite
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark-lite
+    environment:
+      DEBUG: true
+      EXT_BYPASSER_URL: http://flaresolverr:8191
+      EXT_BYPASSER_PATH: /v1
+      EXT_BYPASSER_TIMEOUT: 60000
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads
+
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.dev.tor.yml

@@ -0,0 +1,20 @@
+# Local development - Tor variant
+services:
+  shelfmark-tor-dev:
+    extends:
+      file: ./compose/docker-compose.tor.yml
+      service: shelfmark-tor
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    environment:
+      DEBUG: true
+      USING_TOR: true
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.dev.yml

@@ -1,17 +1,22 @@
 # Local development - builds from source with debug enabled
 services:
-  calibre-web-automated-book-downloader-dev:
+  shelfmark-dev:
     extends:
-      file: ./docker-compose.yml
-      service: calibre-web-automated-book-downloader
+      file: ./compose/docker-compose.yml
+      service: shelfmark
     build:
       context: .
       dockerfile: Dockerfile
-      target: cwa-bd
+      target: shelfmark
+    cap_add:
+      - SYS_PTRACE
     environment:
       DEBUG: true
     volumes:
       - ./.local/config:/config
-      - ./.local/ingest:/cwa-book-ingest
-      - ./.local/log:/var/log/cwa-book-downloader
-      - ./.local/tmp:/tmp/cwa-book-downloader
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      - ./shelfmark:/app/shelfmark:ro
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.extbp.dev.yml

@@ -1,23 +0,0 @@
-# Local development - External bypasser variant
-services:
-  calibre-web-automated-book-downloader-extbp-dev:
-    extends:
-      file: ./docker-compose.extbp.yml
-      service: calibre-web-automated-book-downloader-extbp
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-extbp
-    environment:
-      DEBUG: true
-      EXT_BYPASSER_URL: http://flaresolverr:8191
-      EXT_BYPASSER_PATH: /v1
-      EXT_BYPASSER_TIMEOUT: 60000
-    volumes:
-      - ./.local/config:/config
-      - ./.local/ingest:/cwa-book-ingest
-      - ./.local/log:/var/log/cwa-book-downloader
-      - ./.local/tmp:/tmp/cwa-book-downloader
-
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.extbp.yml

@@ -1,20 +0,0 @@
-# Uses external Cloudflare bypasser (FlareSolverr/ByParr) instead of built-in Selenium
-services:
-  calibre-web-automated-book-downloader-extbp:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-extbp:latest
-    environment:
-      TZ: America/New_York
-      EXT_BYPASSER_URL: http://flaresolverr:8191
-      # UID: 1000
-      # GID: 100
-      # CWA_DB_PATH: /auth/app.db
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      - /path/to/config:/config
-      # - /cwa/config/path/app.db:/auth/app.db:ro
-
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.test-clients.yml

@@ -0,0 +1,181 @@
+# Test stack for download client development
+# Includes shelfmark + all download clients on same network with shared volumes
+#
+# Usage:
+#   docker compose -f docker-compose.test-clients.yml up -d
+#   # Access shelfmark at http://localhost:8084
+#   # Configure clients in Settings > Prowlarr > Download Clients
+#
+# Web UIs:
+#   - shelfmark:    http://localhost:8084
+#   - Prowlarr:     http://localhost:9696 (no auth by default)
+#   - qBittorrent:  http://localhost:8080 (check container logs for temp password)
+#   - Transmission: http://localhost:9091 (admin / admin)
+#   - Deluge:       http://localhost:8112 (password: deluge)
+#   - NZBGet:       http://localhost:6789 (nzbget / tegbzn6789)
+#   - SABnzbd:      http://localhost:8085 (complete setup wizard for API key)
+#   - rTorrent:     http://localhost:8000 (admin / admin - if auth enabled)
+#
+
+services:
+  shelfmark:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    container_name: test-shelfmark
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      # All client configuration is done via Settings UI
+      # Use Docker service names for URLs:
+      #   - qBittorrent: http://qbittorrent:8080
+      #   - Transmission: http://transmission:9091
+      #   - Deluge Web UI: http://deluge:8112
+      #   - NZBGet: http://nzbget:6789
+      #   - SABnzbd: http://sabnzbd:8080
+      #   - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
+    ports:
+      - "8084:8084"
+    volumes:
+      # Config and state
+      - ./.local/test-clients/shelfmark/config:/config
+      - ./.local/test-clients/shelfmark/log:/var/log/shelfmark
+      # Book destination directory (where completed books go)
+      - ./.local/test-clients/books:/books
+      # Staging directory
+      - ./.local/test-clients/tmp:/tmp/shelfmark
+      # CRITICAL: Mount client download directories so shelfmark can access completed files
+      - ./.local/test-clients/downloads:/downloads
+      # Mount source code for hot-reload (no rebuild needed for Python changes)
+      - ./shelfmark:/app/shelfmark:ro
+      # Mount tests for running pytest in container
+      - ./tests:/app/tests:ro
+      - ./pyproject.toml:/app/pyproject.toml:ro
+      # Mount client configs for integration tests to read credentials
+      - ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
+      - ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
+    depends_on:
+      - nzbget
+      - sabnzbd
+      - qbittorrent
+      - transmission
+      - deluge
+      - rtorrent
+    restart: unless-stopped
+
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:latest
+    container_name: test-prowlarr
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/prowlarr/config:/config
+    ports:
+      - "9696:9696"
+    restart: unless-stopped
+
+  nzbget:
+    image: lscr.io/linuxserver/nzbget:latest
+    container_name: test-nzbget
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/nzbget/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/nzbget/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "6789:6789"  # Web UI / JSON-RPC
+    restart: unless-stopped
+
+  sabnzbd:
+    image: lscr.io/linuxserver/sabnzbd:latest
+    container_name: test-sabnzbd
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/sabnzbd/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8085:8080"  # Web UI (external:internal)
+    restart: unless-stopped
+
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: test-qbittorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - WEBUI_PORT=8080
+    volumes:
+      - ./.local/test-clients/qbittorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/qbittorrent/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "8080:8080"  # Web UI / API
+      - "6882:6881"
+      - "6882:6881/udp"
+    restart: unless-stopped
+
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: test-transmission
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - USER=admin
+      - PASS=admin
+    volumes:
+      - ./.local/test-clients/transmission/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "9091:9091"  # Web UI / RPC
+      - "51413:51413"
+      - "51413:51413/udp"
+    restart: unless-stopped
+
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: test-deluge
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - DELUGE_LOGLEVEL=error
+    volumes:
+      - ./.local/test-clients/deluge/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8112:8112"  # Web UI
+      - "58846:58846"  # Daemon RPC
+      - "6881:6881"
+      - "6881:6881/udp"
+    restart: unless-stopped
+
+  rtorrent:
+    image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
+    container_name: test-rtorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/rtorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8000:8000"  # XMLRPC
+      - "8089:8080"  # ruTorrent Web UI
+      - "9000:9000"  # SCGI port
+      - "50000:50000"  # Incoming connections
+      - "6881:6881/udp"
+    restart: unless-stopped

docker-compose.tor.dev.yml

@@ -1,17 +0,0 @@
-# Local development - Tor variant
-services:
-  calibre-web-automated-book-downloader-tor-dev:
-    extends:
-      file: ./docker-compose.tor.yml
-      service: calibre-web-automated-book-downloader-tor
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-tor
-    environment:
-      DEBUG: true
-    volumes:
-      - ./.local/config:/config
-      - ./.local/ingest:/cwa-book-ingest
-      - ./.local/log:/var/log/cwa-book-downloader
-      - ./.local/tmp:/tmp/cwa-book-downloader

docker-compose.tor.yml

@@ -1,19 +0,0 @@
-# Routes all traffic through Tor - requires NET_ADMIN capability
-services:
-  calibre-web-automated-book-downloader-tor:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest
-    environment:
-      FLASK_PORT: 8084
-      TZ: America/New_York
-      USING_TOR: true
-      # CWA_DB_PATH: /auth/app.db
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      - /path/to/config:/config
-      # - /cwa/config/path/app.db:/auth/app.db:ro

docker-compose.yml

@@ -1,15 +0,0 @@
-services:
-  calibre-web-automated-book-downloader:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
-    container_name: calibre-web-automated-book-downloader
-    environment:
-      TZ: America/New_York
-      # UID: 1000
-      # GID: 100
-      # CWA_DB_PATH: /auth/app.db
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest # This is where the books will be downloaded and ingested by your book management application
-      - /path/to/config:/config # Configuration files and database

docs/configuration.md

@@ -0,0 +1,3 @@
+# Configuration
+
+TODO

docs/dev/index.md

@@ -0,0 +1,3 @@
+# Developer Documentation
+
+TODO

docs/dev/plugin-settings.md

@@ -16,7 +16,7 @@ The settings system uses a decorator-based registration pattern. Plugins registe
 Add settings to your plugin in 3 steps:
 
 ```python
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     TextField,
     PasswordField,
@@ -249,7 +249,7 @@ The field will be greyed out with the specified reason when the condition is met
 Register a group to organize related settings tabs in the sidebar:
 
 ```python
-from cwa_book_downloader.core.settings_registry import register_group
+from shelfmark.core.settings_registry import register_group
 
 # Register a group (do this once, usually in a central config file)
 register_group(
@@ -289,7 +289,7 @@ The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdire
 Use the `config` singleton to read setting values in your plugin code:
 
 ```python
-from cwa_book_downloader.core.config import config
+from shelfmark.core.config import config
 
 # Get a setting value with default fallback
 api_key = config.get("MY_PLUGIN_API_KEY", "")
@@ -312,13 +312,13 @@ The config singleton:
 Here's a complete example for a metadata provider plugin:
 
 ```python
-# cwa_book_downloader/metadata_providers/my_provider.py
+# shelfmark/metadata_providers/my_provider.py
 
-from cwa_book_downloader.metadata_providers.base import (
+from shelfmark.metadata_providers.base import (
     MetadataProvider,
     register_provider,
 )
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -326,7 +326,7 @@ from cwa_book_downloader.core.settings_registry import (
     CheckboxField,
     ActionButton,
 )
-from cwa_book_downloader.core.config import config
+from shelfmark.core.config import config
 
 
 def _test_connection():
@@ -422,15 +422,15 @@ class MyProvider(MetadataProvider):
 Here's a complete example for a release source plugin:
 
 ```python
-# cwa_book_downloader/release_sources/my_source.py
+# shelfmark/release_sources/my_source.py
 
-from cwa_book_downloader.release_sources.base import (
+from shelfmark.release_sources.base import (
     ReleaseSource,
     DownloadHandler,
     register_source,
     register_handler,
 )
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -439,7 +439,7 @@ from cwa_book_downloader.core.settings_registry import (
     SelectField,
     ActionButton,
 )
-from cwa_book_downloader.core.config import config
+from shelfmark.core.config import config
 
 
 def _test_source():

docs/dev/release-sources-plugin-guide.md

@@ -1,6 +1,6 @@
 # Release Sources Plugin Development Guide
 
-This guide explains how to create custom release source plugins for the CWA Book Downloader. The plugin system allows you to add new sources for searching and downloading books while integrating seamlessly with the existing queue, progress reporting, and settings infrastructure.
+This guide explains how to create custom release source plugins for the Shelfmark. The plugin system allows you to add new sources for searching and downloading books while integrating seamlessly with the existing queue, progress reporting, and settings infrastructure.
 
 ## Table of Contents
 
@@ -43,7 +43,7 @@ The release sources system is built around two core interfaces:
 - **DownloadHandler**: Executes the actual download with progress reporting
 
 This separation allows:
-- Different search sources (Anna's Archive, Prowlarr, IRC, etc.)
+- Different search sources (Direct Download, Prowlarr, IRC, etc.)
 - Different download protocols (HTTP, torrent, usenet, etc.)
 - Shared queue and progress infrastructure
 
@@ -77,21 +77,21 @@ class MyHandler(DownloadHandler): ...
 
 ## Quick Start
 
-Create a new file at `cwa_book_downloader/release_sources/my_source.py`:
+Create a new file at `shelfmark/release_sources/my_source.py`:
 
 ```python
 from typing import Callable, List, Optional
 from threading import Event
 
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     Release,
     ReleaseSource,
     DownloadHandler,
     register_source,
     register_handler,
 )
-from cwa_book_downloader.metadata_providers import BookMetadata
-from cwa_book_downloader.core.models import DownloadTask
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.core.models import DownloadTask
 
 
 @register_source("my_source")
@@ -123,11 +123,11 @@ class MyHandler(DownloadHandler):
         return False  # Cancellation via cancel_flag
 ```
 
-Register the import in `cwa_book_downloader/release_sources/__init__.py`:
+Register the import in `shelfmark/release_sources/__init__.py`:
 
 ```python
 # At the bottom of the file
-from cwa_book_downloader.release_sources import my_source  # noqa: F401, E402
+from shelfmark.release_sources import my_source  # noqa: F401, E402
 ```
 
 ---
@@ -139,8 +139,8 @@ The `ReleaseSource` abstract base class defines the search interface:
 ```python
 from abc import ABC, abstractmethod
 from typing import List
-from cwa_book_downloader.metadata_providers import BookMetadata
-from cwa_book_downloader.release_sources import Release, ReleaseColumnConfig
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.release_sources import Release, ReleaseColumnConfig
 
 class ReleaseSource(ABC):
     """Interface for searching a release source."""
@@ -237,7 +237,7 @@ The `DownloadHandler` abstract base class defines the download interface:
 from abc import ABC, abstractmethod
 from typing import Callable, Optional
 from threading import Event
-from cwa_book_downloader.core.models import DownloadTask
+from shelfmark.core.models import DownloadTask
 
 class DownloadHandler(ABC):
     """Interface for executing downloads from a source."""
@@ -371,7 +371,7 @@ class Release:
     info_url: Optional[str] = None      # Link to tracker/info page
 
     protocol: Optional[str] = None   # "http", "torrent", "usenet"
-    indexer: Optional[str] = None    # Display name: "Anna's Archive", "MyAnonamouse"
+    indexer: Optional[str] = None    # Display name: "Direct Download", "My Indexer"
     seeders: Optional[int] = None    # For torrents
 
     extra: Dict = field(default_factory=dict)  # Source-specific metadata
@@ -458,7 +458,7 @@ class BookMetadata:
 ### Decorators
 
 ```python
-from cwa_book_downloader.release_sources import register_source, register_handler
+from shelfmark.release_sources import register_source, register_handler
 
 @register_source("my_source")
 class MySource(ReleaseSource):
@@ -472,7 +472,7 @@ class MyHandler(DownloadHandler):
 ### Registry Functions
 
 ```python
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     get_source,
     get_handler,
     list_available_sources,
@@ -497,8 +497,8 @@ available = list_available_sources()
 
 ```python
 # At the bottom of __init__.py
-from cwa_book_downloader.release_sources import direct_download  # noqa: F401, E402
-from cwa_book_downloader.release_sources import my_source  # noqa: F401, E402
+from shelfmark.release_sources import direct_download  # noqa: F401, E402
+from shelfmark.release_sources import my_source  # noqa: F401, E402
 ```
 
 The `noqa` comments suppress linter warnings about unused imports.
@@ -510,7 +510,7 @@ The `noqa` comments suppress linter warnings about unused imports.
 Register plugin settings using the settings registry decorator:
 
 ```python
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -621,7 +621,7 @@ def test_my_source_connection():
 ### Reading Settings
 
 ```python
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     get_setting_value,
     is_value_from_env,
     load_config_file,
@@ -658,7 +658,7 @@ When a value comes from an ENV var, the UI shows a "locked" badge and the field
 Customize how releases are displayed in the release modal by overriding `get_column_config()`:
 
 ```python
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     ReleaseColumnConfig,
     ColumnSchema,
     ColumnRenderType,
@@ -907,7 +907,7 @@ from pathlib import Path
 from typing import Callable, Dict, List, Optional
 from threading import Event
 
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     Release,
     ReleaseSource,
     DownloadHandler,
@@ -921,10 +921,10 @@ from cwa_book_downloader.release_sources import (
     LeadingCellConfig,
     LeadingCellType,
 )
-from cwa_book_downloader.metadata_providers import BookMetadata
-from cwa_book_downloader.core.models import DownloadTask
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -934,7 +934,7 @@ from cwa_book_downloader.core.settings_registry import (
     ActionButton,
     load_config_file,
 )
-from cwa_book_downloader.config.env import INGEST_DIR
+from shelfmark.config.env import INGEST_DIR
 
 logger = setup_logger(__name__)
 
@@ -1421,7 +1421,7 @@ PasswordField(
 Use the project's logger for consistent output:
 
 ```python
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -1437,13 +1437,13 @@ logger.error("Failures")                 # Unrecoverable errors
 
 When creating a new plugin:
 
-1. Create `cwa_book_downloader/release_sources/my_plugin.py`
+1. Create `shelfmark/release_sources/my_plugin.py`
 2. Implement `ReleaseSource` subclass with `@register_source("my_plugin")`
 3. Implement `DownloadHandler` subclass with `@register_handler("my_plugin")`
 4. Add settings with `@register_settings("my_plugin", ...)`
-5. Add import to `cwa_book_downloader/release_sources/__init__.py`:
+5. Add import to `shelfmark/release_sources/__init__.py`:
    ```python
-   from cwa_book_downloader.release_sources import my_plugin  # noqa: F401, E402
+   from shelfmark.release_sources import my_plugin  # noqa: F401, E402
    ```
 6. Test `is_available()` returns `True` when configured
 7. Test search returns valid `Release` objects

docs/index.md

@@ -0,0 +1,3 @@
+# Shelfmark Documentation
+
+TODO

docs/installation.md

@@ -0,0 +1,3 @@
+# Installation
+
+TODO

docs/reverse-proxy.md

@@ -0,0 +1,35 @@
+# Reverse Proxy & Subpath Hosting
+
+Shelfmark can run behind a reverse proxy at the root path (recommended) or
+under a subpath like `/shelfmark`.
+
+## Subpath setup
+
+1) Set the base path in Shelfmark:
+- UI: Settings → Advanced → Base Path
+- Env var: `URL_BASE=/shelfmark`
+
+2) Configure your reverse proxy to forward the subpath to Shelfmark and
+**strip the prefix** before sending to the backend. The proxy must also allow
+WebSocket upgrades for Socket.IO.
+
+Example (Nginx-style):
+
+```
+location /shelfmark/ {
+  proxy_pass http://shelfmark:8084/;
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection "upgrade";
+}
+```
+
+Notes:
+- Use a trailing slash on the `location` and `proxy_pass` to ensure the
+  `/shelfmark` prefix is removed.
+- Health checks still work at `/api/health` without the subpath.
+
+## Root path setup
+
+If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`),
+leave `URL_BASE` empty. This is the simplest option.

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+# Troubleshooting
+
+TODO

docs/url-search-parameters.md

@@ -61,7 +61,7 @@ Some parameters support multiple values by repeating the parameter:
 
 ### Direct Download Mode (default)
 
-All parameters are used to filter results from Anna's Archive.
+All parameters are used to filter results from the direct download source.
 
 ### Universal Mode
 

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/cwa-book-downloader
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/cwa-bd_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+
+    # Cleanup any existing files or folders in the log directory
+    rm -rf "$LOG_DIR"/*
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,10 +56,16 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
@@ -28,34 +78,57 @@ if [ "$TZ" ]; then
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 fi
 
-# Set UID if not set
-if [ -z "$UID" ]; then
-    UID=1000
+# Determine user ID with proper precedence:
+# 1. PUID (LinuxServer.io standard - recommended)
+# 2. UID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+#
+# Note: $UID is a bash builtin that's always set. We use `printenv` to detect
+# if UID was explicitly set as an environment variable (e.g., via docker-compose).
+if [ -n "$PUID" ]; then
+    RUN_UID="$PUID"
+    echo "Using PUID=$RUN_UID"
+elif printenv UID >/dev/null 2>&1; then
+    RUN_UID="$(printenv UID)"
+    echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)"
+else
+    RUN_UID=1000
+    echo "Using default UID=$RUN_UID"
 fi
 
-# Set GID if not set
-if [ -z "$GID" ]; then
-    GID=100
+# Determine group ID with proper precedence:
+# 1. PGID (LinuxServer.io standard - recommended)
+# 2. GID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+if [ -n "$PGID" ]; then
+    RUN_GID="$PGID"
+    echo "Using PGID=$RUN_GID"
+elif [ -n "$GID" ]; then
+    RUN_GID="$GID"
+    echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)"
+else
+    RUN_GID=1000
+    echo "Using default GID=$RUN_GID"
 fi
 
-if ! getent group "$GID" >/dev/null; then
-    echo "Adding group $GID with name appuser"
-    groupadd -g "$GID" appuser
+if ! getent group "$RUN_GID" >/dev/null; then
+    echo "Adding group $RUN_GID with name appuser"
+    groupadd -g "$RUN_GID" appuser
 fi
 
 # Create user if it doesn't exist
-if ! id -u "$UID" >/dev/null 2>&1; then
-    echo "Adding user $UID with name appuser"
-    useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser
+if ! id -u "$RUN_UID" >/dev/null 2>&1; then
+    echo "Adding user $RUN_UID with name appuser"
+    useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser
 fi
 
 # Get username for the UID (whether we just created it or it existed)
-USERNAME=$(getent passwd "$UID" | cut -d: -f1)
-echo "Username for UID $UID is $USERNAME"
+USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1)
+echo "Username for UID $RUN_UID is $USERNAME"
 
 test_write() {
     folder=$1
-    test_file=$folder/calibre-web-automated-book-downloader_TEST_WRITE
+    test_file=$folder/shelfmark_TEST_WRITE
     mkdir -p $folder
     (
         echo 0123456789_TEST | sudo -E -u "$USERNAME" HOME=/app tee $test_file > /dev/null
@@ -84,7 +157,16 @@ make_writable() {
     else
         echo "Folder $folder is not writable, changing ownership"
         change_ownership $folder
-        chmod g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        chmod -R g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+    fi
+    # Fix any misowned subdirectories/files (e.g., from previous runs as root)
+    if [ -d "$folder" ]; then
+        misowned_count=$(find "$folder" -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) 2>/dev/null | wc -l)
+        if [ "$misowned_count" -gt 0 ]; then
+            echo "Fixing ownership of $misowned_count files/directories in $folder"
+            find "$folder" -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+                -exec chown "$RUN_UID:$RUN_GID" {} \; 2>/dev/null || true
+        fi
     fi
     test_write $folder || echo "Failed to test write to ${folder}, continuing..."
 }
@@ -93,23 +175,88 @@ make_writable() {
 change_ownership() {
   folder=$1
   mkdir -p $folder
-  echo "Changing ownership of $folder to $USERNAME:$GID"
-  chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
-  chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
+  echo "Changing ownership of $folder to $USERNAME:$RUN_GID"
+  chown -R "${RUN_UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
+  chown -R ":${RUN_GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
 }
 
 change_ownership /app
-change_ownership /var/log/cwa-book-downloader
-change_ownership /tmp/cwa-book-downloader
+change_ownership /var/log/shelfmark
+change_ownership /tmp/shelfmark
+
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
+
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
+fi
 
 # Test write to all folders
-make_writable /cwa-book-ingest
+make_writable ${CONFIG_DIR:-/config}
+make_writable ${INGEST_DIR:-/books}
+
+# Fix permissions on directories configured in settings
+echo "Checking for additional configured directories..."
+if [ -f /app/scripts/fix_permissions.py ]; then
+    configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
+    if [ -n "$configured_dirs" ]; then
+        echo "$configured_dirs" | while read -r dir; do
+            if [ -n "$dir" ] && [ -d "$dir" ]; then
+                echo "Checking configured directory: $dir"
+                make_writable "$dir"
+            fi
+        done
+    fi
+fi
+
+# Fallback to root if config dir is still not writable (common on NAS/Unraid after upgrade from v0.4.0)
+CONFIG_PATH=${CONFIG_DIR:-/config}
+set +e
+test_write "$CONFIG_PATH" >/dev/null 2>&1
+config_ok=$?
+set -e
+
+if [ $config_ok -ne 0 ] && [ "$RUN_UID" != "0" ]; then
+    config_owner=$(stat -c '%u' "$CONFIG_PATH" 2>/dev/null || echo "unknown")
+    if [ "$config_owner" = "0" ]; then
+        echo ""
+        echo "========================================================"
+        echo "WARNING: Permission issue detected!"
+        echo ""
+        echo "Config directory is owned by root but PUID=$RUN_UID."
+        echo "This typically happens after upgrading from v0.4.0 where"
+        echo "PUID/PGID settings were not respected."
+        echo ""
+        echo "Falling back to running as root to prevent data loss."
+        echo ""
+        echo "To fix this permanently, run on your HOST machine:"
+        echo "  chown -R $RUN_UID:$RUN_GID /path/to/config"
+        echo ""
+        echo "Then restart the container."
+        echo "========================================================"
+        echo ""
+        RUN_UID=0
+        RUN_GID=0
+        USERNAME=root
+    fi
+fi
 
 # Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
 # upgrades work reliably on customer machines.
 # Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
 gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
-command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} cwa_book_downloader.main:app"
+command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
 
 # If DEBUG and not using an external bypass
 if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
@@ -165,18 +312,24 @@ if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
 fi
 
-# Hacky way to verify /tmp has at least 1MB of space and is writable/readable
+# Verify /tmp has at least 1MB of space and is writable/readable
 echo "Verifying /tmp has enough space"
-rm -f /tmp/test.cwa-bd
-for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd
-sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))")
-[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1)
-rm /tmp/test.cwa-bd
+rm -f /tmp/test.shelfmark
+if dd if=/dev/zero of=/tmp/test.shelfmark bs=1M count=1 2>/dev/null && \
+   [ "$(wc -c < /tmp/test.shelfmark)" -eq 1048576 ]; then
+    rm -f /tmp/test.shelfmark
+    echo "Success: /tmp is writable and readable"
+else
+    echo "Failure: /tmp is not writable or has insufficient space"
+    exit 1
+fi
 
 echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
+# Set umask for file permissions (default: 0022 = files 644, dirs 755)
+UMASK_VALUE=${UMASK:-0022}
+echo "Setting umask to $UMASK_VALUE"
+umask $UMASK_VALUE
 
+stop_file_logging
 exec sudo -E -u "$USERNAME" HOME=/app $command

genDebug.sh

@@ -2,8 +2,8 @@
 
 # Set up log paths
 LOG_ROOT=${LOG_ROOT:-"/var/log"}
-LOG_DIR="$LOG_ROOT/cwa-book-downloader"
-OUTPUT_FILE_NAME="cwa-book-downloader-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
+LOG_DIR="$LOG_ROOT/shelfmark"
+OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
 OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
 
 # Create LOG_DIR if it doesn't exist
@@ -100,14 +100,14 @@ echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
 ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt" 2>&1
+echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
+ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt" 2>&1
+echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt" 2>&1
+echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 
 # Check Iptables (NAT)

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "shelfmark"
+version = "0.1.0"
+description = "Shelfmark - Book Downloader"
+requires-python = ">=3.10"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--tb=short",
+]
+markers = [
+    "integration: marks tests that require running services (deselect with '-m \"not integration\"')",
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks end-to-end tests that require the full application stack",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+ignore_missing_imports = true

readme.md

@@ -1,365 +1,258 @@
-# 📚 Calibre-Web-Automated-Book-Downloader
+# 📚 Shelfmark: Book Downloader
 
-<img src="src/frontend/public/logo.png" alt="Calibre-Web Automated Book Downloader" width="200">
+Formerly *Calibre Web Automated Book Downloader (CWABD)*
 
-An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
+<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
+
+Shelfmark is a unified web interface for searching and aggregating books and audiobook downloads from multiple sources - all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library.
+
+**Fully standalone** - no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated), [Booklore](https://github.com/booklore-app/booklore) or [Audiobookshelf](https://github.com/advplyr/audiobookshelf) for automatic import.
 
 ## ✨ Features
 
-- 🌐 User-friendly web interface for book search and download
-- 🔄 Automated download to your specified ingest folder
-- 🔌 Seamless integration with Calibre-Web-Automated
-- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr)
-- 🛡️ Cloudflare bypass capability for reliable downloads
-- 🐳 Docker-based deployment for quick setup
+- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
+- **Multiple sources** - Popular archive websites, Torrent, Usenet and IRC download support
+- **Audiobook support** - Full audiobook search and download with dedicated processing
+- **Real-Time Progress** - Unified download queue with live status updates across all sources
+- **Two Search Modes**:
+  - **Direct** - Search popular web sources
+  - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
+- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
 
 ## 🖼️ Screenshots
 
-![Homescreen](README_images/homescreen.png 'Homescreen')
+**Home screen**
+![Home screen](README_images/homescreen.png 'Home screen')
 
+**Search results**
 ![Search results](README_images/search-results.png 'Search results')
 
-![Active downloads](README_images/downloads.png 'Active downloads')
+**Multi-source downloads**
+![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
+
+**Download queue**
+![Download queue](README_images/downloads.png 'Download queue')
 
 ## 🚀 Quick Start
 
 ### Prerequisites
 
-- Docker
-- Docker Compose
-- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended)
+- Docker & Docker Compose
 
-### Installation Steps
-
-1. Get the docker-compose.yml:
+### Installation
 
+1. Download the [docker-compose file](compose/docker-compose.yml):
    ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml
+   curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
    ```
 
 2. Start the service:
-
    ```bash
    docker compose up -d
    ```
 
-3. Access the web interface at `http://localhost:8084`
+3. Open `http://localhost:8084`
 
-## ⚙️ Configuration
+That's it! Configure settings through the web interface as needed.
 
-### Environment Variables
-
-#### Application Settings
-
-| Variable          | Description             | Default Value      |
-| ----------------- | ----------------------- | ------------------ |
-| `FLASK_PORT`      | Web interface port      | `8084`             |
-| `FLASK_HOST`      | Web interface binding   | `0.0.0.0`          |
-| `DEBUG`           | Debug mode toggle       | `false`            |
-| `INGEST_DIR`      | Book download directory | `/cwa-book-ingest` |
-| `TZ`              | Container timezone      | `UTC`              |
-| `UID`             | Runtime user ID         | `1000`             |
-| `GID`             | Runtime group ID        | `100`              |
-| `CWA_DB_PATH`     | Calibre-Web's database  | None               |
-| `ENABLE_LOGGING`  | Enable log file         | `true`             |
-| `LOG_LEVEL`       | Log level to use        | `info`             |
-| `SESSION_COOKIE_SECURE` | Secure cookie enforcement - Use for HTTPS connections only | `false` |
-| `CALIBRE_WEB_URL` | Custom WebUI library link | None |
-| `BYPASS_WARMUP_ON_CONNECT` | Warm up Cloudflare bypasser when first client connects | `true` |
-
-If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
-
-Set `CALIBRE_WEB_URL` to your Calibre-Web / Booklore base URL. A ‘Go to library’ button will appear in the Web UI for quick access while downloading, and it also provides library access when CWA-BD is installed as a mobile PWA.
-
-If logging is enabled, log folder default location is `/var/log/cwa-book-downloader`
-Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages.
-
-Note that if using TOR, the TZ will be calculated automatically based on IP.
-
-#### Download Settings
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `MAX_RETRY`            | Maximum retry attempts                                    | `3`                               |
-| `DEFAULT_SLEEP`        | Retry delay (seconds)                                     | `5`                               |
-| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds)                           | `5`                               |
-| `SUPPORTED_FORMATS`    | Supported book formats                                    | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
-| `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
-| `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
-| `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
-| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
-| `ALLOW_USE_WELIB`       | Allow usage of welib for downloading books if found there | `true`                            |
-
-If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
-
-Use the following environment variables to set specific folders in which to download 
-different content types (Book, Magazine, Comic, etc.):
-
-| Variable                        | Description                    | Default Value |
-|---------------------------------|--------------------------------|---------------|
-| `INGEST_DIR_BOOK_FICTION`       | Book (fiction) folder name     | ``            |
-| `INGEST_DIR_BOOK_NON_FICTION`   | Book (non-fiction) folder name | ``            |
-| `INGEST_DIR_BOOK_UNKNOWN`       | Book (unknown) folder name     | ``            |
-| `INGEST_DIR_MAGAZINE`           | Magazine folder name           | ``            |
-| `INGEST_DIR_COMIC_BOOK`         | Comic book folder name         | ``            |
-| `INGEST_DIR_AUDIOBOOK`          | Audiobook folder name          | ``            |
-| `INGEST_DIR_STANDARDS_DOCUMENT` | Standards document folder name | ``            |
-| `INGEST_DIR_MUSICAL_SCORE`      | Musical score folder name      | ``            |
-
-If no specific path is set for a content type the default is `INGEST_DIR`.  
-Remember to map the specified paths to where your instance of Calibre-Web-Automated (CWA) will find them, e.g.:  
-```
-volumes:
-    - /tmp/data/calibre-web/comicbook-ingest:/cwa-comicbook-ingest
-```
-if `INGEST_DIR_COMIC_BOOK=/cwa-comicbook-ingest` and your CWA is configured to use `/tmp/data/calibre-web/comicbook-ingest` 
-for comic books.
-
-
-#### AA 
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
-
-If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
-If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.
-
-#### Network Settings
-
-| Variable               | Description                     | Default Value           |
-| ---------------------- | ------------------------------- | ----------------------- |
-| `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
-| `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
-| `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | DNS configuration               | `auto`                  |
-| `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
-
-**Proxy Configuration**
-
-For proxy configuration, you can specify URLs in the following format:
-```bash
-# Basic proxy
-HTTP_PROXY=http://proxy.example.com:8080
-HTTPS_PROXY=http://proxy.example.com:8080
-
-# Proxy with authentication
-HTTP_PROXY=http://username:password@proxy.example.com:8080
-HTTPS_PROXY=http://username:password@proxy.example.com:8080
-```
-
-**DNS Configuration**
-
-The `CUSTOM_DNS` setting controls how DNS resolution works. By default, it is set to `auto` which provides automatic failover for reliable connectivity.
-
-**Auto Mode (Default)**
-
-When `CUSTOM_DNS=auto`, the application starts with your system's default DNS. If DNS resolution fails, it automatically rotates through alternative providers using DNS over HTTPS (DoH):
-
-1. System DNS (initial)
-2. Cloudflare (1.1.1.1)
-3. Google (8.8.8.8)
-4. Quad9 (9.9.9.9)
-5. OpenDNS (208.67.222.222)
-
-This automatic rotation helps bypass ISP-level blocks and DNS issues without any manual configuration.
-
-**Manual DNS Configuration**
-
-If you prefer to use a specific DNS configuration, you can override the auto behavior:
-
-1. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS (8.8.8.8, 8.8.4.4)
-   - `quad9` - Quad9 DNS (9.9.9.9, 149.112.112.112)
-   - `cloudflare` - Cloudflare DNS (1.1.1.1, 1.0.0.1)
-   - `opendns` - OpenDNS (208.67.222.222, 208.67.220.220)
-
-2. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
-   - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses
-
-When using preset providers, you can optionally enable DNS over HTTPS with `USE_DOH=true`:
-```bash
-CUSTOM_DNS=cloudflare
-USE_DOH=true
-```
-
-Note: When using custom IP addresses, the `USE_DOH` flag is ignored since DoH requires a known provider endpoint.
-
-#### Custom configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `CUSTOM_SCRIPT`        | Path to an executable script that tuns after each download  | ``                      |
-
-If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation.
-
-The script is called with the full path of the downloaded file as its argument. Important notes:
-- The script must preserve the original filename for proper processing
-- The file can be modified or even deleted if needed
-- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted)
-
-You can specify these configuration in this format :
-```
-environment:
-  - CUSTOM_SCRIPT=/scripts/process-book.sh
-
-volumes:
-  - local/scripts/custom_script.sh:/scripts/process-book.sh
-```
-
-### Volume Configuration
+### Volume Setup
 
 ```yaml
 volumes:
-  - /your/local/path:/cwa-book-ingest
-  - /cwa/config/path/app.db:/auth/app.db:ro
+  - /your/config/path:/config # Config, database, and artwork cache directory
+  - /your/download/path:/books # Downloaded books
+  - /client/path:/client/path # Optional: For Torrent/Usenet downloads, match your client directory exactly. 
 ```
-**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777
 
-Mount should align with your Calibre-Web-Automated ingest folder.
+> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
 
-## Variants:
+> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
 
-### 🧅 Tor Variant
+## ⚙️ Configuration
 
-This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions.
+### Search Modes
 
-To use the Tor variant:
+**Direct** (default)
+- Works out of the box, no setup required
+- Searches a huge library of books directly
+- Returns downloadable releases immediately
 
-1.  Get the Tor-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.tor.yml up -d
-    ```
+**Universal**
+- Cleaner search results via metadata providers (Hardcover is recommended)
+- Aggregates releases from multiple configured sources
+- Full Audiobook support
+- Requires manual setup (API keys, additional sources)
 
-**Important Considerations for Tor:**
+### Environment Variables
 
-*   **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying.
-*   **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set.
-*   **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor.
+Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
 
-### External Cloudflare resolver variant
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FLASK_PORT` | Web interface port | `8084` |
+| `INGEST_DIR` | Book download directory | `/books` |
+| `TZ` | Container timezone | `UTC` |
+| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` |
+| `SEARCH_MODE` | `direct` or `universal` | `direct` |
+| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
 
-This variant allows the application to use an external service to bypass Cloudflare protection, instead of relying on the built-in bypasser. This is useful if you already have a dedicated Cloudflare resolver (such as [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) or compatible services like [ByParr](https://github.com/ThePhaseless/Byparr)) running elsewhere.
+See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
 
-#### How it works:
+Some of the additional options available in Settings:
+- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
+- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
+- **IRC** - Add details for IRC book sources and download directly from the UI
+- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
+- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
+- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
+- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
+- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
 
-- When enabled, all requests that require Cloudflare bypass are sent to your external resolver service.
-- The application communicates with the resolver using its API.
+## 🐳 Docker Variants
 
-#### Configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `EXT_BYPASSER_URL`     | The full URL of your external resolver (required)           |                         |
-| `EXT_BYPASSER_PATH`    | API path for the resolver (usually `/v1`)                   | `/v1`                   |
-| `EXT_BYPASSER_TIMEOUT` | Timeout for page loading (in milliseconds)                  | `60000`                 |
-
-#### Important
-
-This feature follows the same configuration of the built-in Cloudflare bypasser, so you should turn on the `USE_CF_BYPASS` configuration to enable it.
-
-#### To use the External Cloudflare resolver variant:
-
-1.  Get the extbp-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.extbp.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.extbp.yml up -d
-    ```
-
-#### Compatibility:
-This feature is designed to work with any resolver that implements the `FlareSolverr` API schema, including `ByParr` and similar projects.
-
-#### Internal vs External Bypasser
-
-The **internal bypasser** (default) is custom-designed for this application's specific needs. It handles session management, cookie persistence, and retry logic optimized for book downloading workflows. For most users, this provides the most reliable experience out of the box.
-
-The **external bypasser** is better suited if you:
-- Already run FlareSolverr/ByParr for other services and want to consolidate
-- Need to share bypass infrastructure across multiple applications
-- Want to offload browser automation to a dedicated, more powerful container
-
-If you're unsure which to use, start with the default internal bypasser.
-
-## 🏗️ Architecture
-
-The application consists of a Flask backend with a React-based frontend:
-
-### Backend
-- **Flask Application**: Python-based backend (`app.py`, `backend.py`) providing REST API and WebSocket support
-- **Download Manager**: Handles book search, download requests, and queue management (`downloader.py`, `book_manager.py`)
-- **Network Layer**: Cloudflare bypass and proxy support (`cloudflare_bypasser.py`, `network.py`)
-
-### Frontend
-- **React + TypeScript**: Modern web interface built with Vite (`src/frontend`)
-- **Real-time Updates**: WebSocket integration for live download status
-- **Responsive UI**: TailwindCSS-based design for mobile and desktop
-
-For frontend development, use the provided Makefile:
+### Standard
 ```bash
-make install  # Install dependencies
-make dev      # Start development server
-make build    # Build for production
-```
-If you run the docker compose file, the frontend will be built and served automatically. But if you run the frontend dev server it will supercede the docker compose frontend.
-
-## 🏥 Health Monitoring
-
-Built-in health checks monitor:
-
-- Web interface availability
-- Download service status
-- Cloudflare bypass service connection
-
-Checks run every 30 seconds with a 30-second timeout and 3 retries.
-You can enable by adding this to your compose :
-```
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD curl -s http://localhost:8084/api/status || exit 1
+docker compose up -d
 ```
 
-## 📝 Logging
+The full-featured image with built-in Cloudflare bypass.
 
-Logs are available in:
+#### Enable Tor Routing
+Routes all traffic through Tor for enhanced privacy:
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
+docker compose -f docker-compose.tor.yml up -d
+```
 
-- Container: `/var/logs/cwa-book-downloader.log`
-- Docker logs: Access via `docker logs`
+**Notes:**
+- Requires `NET_ADMIN` and `NET_RAW` capabilities
+- Timezone is auto-detected from Tor exit node
+- Custom DNS/proxy settings are ignored when Tor is active
 
-## 🤝 Contributing
+### Lite
+A smaller image without the built-in Cloudflare bypasser. Ideal for:
 
-Contributions are welcome! Feel free to submit a Pull Request.
+- **External bypassers** - Already running FlareSolverr or ByParr for other services
+- **Fast downloads** - Using fast download sources
+- **Alternative sources only** - Exclusively using Prowlarr, IRC, or other sources
+- **Audiobooks** - Using Shelfmark exclusively for audiobooks
 
-## 📄 License
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
+docker compose -f docker-compose.lite.yml up -d
+```
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
 
-## ⚠️ Important Disclaimers
+## 🔐 Authentication
+
+Authentication is optional but recommended for shared or exposed instances. Three authentication methods are available in Settings:
+
+**1. Single Username/Password**
+
+**2. Proxy (Forward) Authentication**
+
+Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
+
+**3. Calibre-Web Database**
+
+If you're running Calibre-Web, you can reuse its user database by mounting it:
+
+```yaml
+volumes:
+  - /path/to/calibre-web/app.db:/auth/app.db:ro
+```
+
+## Health Monitoring
+
+The application exposes a health endpoint at `/api/health` (no authentication required). Add a health check to your compose:
+
+```yaml
+healthcheck:
+  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/health"]
+  interval: 30s
+  timeout: 30s
+  retries: 3
+```
+
+## Logging
+
+Logs are available via:
+- `docker logs <container-name>`
+- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
+
+Log level is configurable via Settings or `LOG_LEVEL` environment variable.
+
+## Development
+
+```bash
+# Frontend development
+make install     # Install dependencies
+make dev         # Start Vite dev server (localhost:5173)
+make build       # Production build
+make typecheck   # TypeScript checks
+
+# Backend (Docker)
+make up          # Start backend via docker-compose.dev.yml
+make down        # Stop services
+make refresh     # Rebuild and restart
+make restart     # Restart container
+```
+
+The frontend dev server proxies to the backend on port 8084.
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      Web Interface                          │
+│                 (React + TypeScript + Vite)                 │
+├─────────────────────────────────────────────────────────────┤
+│                      Flask Backend                          │
+│                   (REST API + WebSocket)                    │
+├───────────────────┬─────────────────────┬───────────────────┤
+│ Metadata Providers│   Download Queue    │  Cloudflare       │
+│                   │   & Orchestrator    │  Bypass           │
+├───────────────────┼─────────────────────┼───────────────────┤
+│ • Hardcover       │ • Task scheduling   │ • Internal        │
+│ • Open Library    │ • Progress tracking │ • External        │
+│                   │ • Retry logic       │   (FlareSolverr)  │
+├───────────────────┴─────────────────────┴───────────────────┤
+│                     Release Sources                         │
+├─────────────────────────────────────────────────────────────┤
+│ • Direct Download (Web Sources → Mirrors → Fallbacks)       │
+├─────────────────────────────────────────────────────────────┤
+│                     Network Layer                           │
+├─────────────────────────────────────────────────────────────┤
+│ • Auto DNS rotation  • Mirror failover  • Resume support    │
+└─────────────────────────────────────────────────────────────┘
+```
+
+The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
+
+## Contributing
+
+Contributions are welcome! Please file issues or submit pull requests on GitHub.
+
+> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR!
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## ⚠️ Disclaimers
 
 ### Copyright Notice
 
-While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for:
-
+This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
 - Ensuring they have the right to download requested materials
 - Respecting copyright laws and intellectual property rights
 - Using the tool in compliance with their local regulations
 
-### Duplicate Downloads Warning
+### Library Integration
 
-Please note that the current version:
+Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
 
-- Does not check for existing files in the download directory
-- Does not verify if books already exist in your Calibre database
-- Exercise caution when requesting multiple books to avoid duplicates
-
-## 💬 Support
-
-For issues or questions, please file an issue on the GitHub repository.
+## Support
 
+For issues or questions, please [file an issue](https://github.com/calibrain/shelfmark/issues) on GitHub.

requirements-base.txt

@@ -12,3 +12,5 @@ gevent-websocket
 psutil
 emoji
 rarfile
+qbittorrent-api
+transmission-rpc

requirements-shelfmark.txt

@@ -1,4 +1,4 @@
 pyvirtualdisplay
 pyautogui
-seleniumbase>=4.41.1
+seleniumbase>=4.45.6
 python-xlib

scripts/fix_permissions.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fix permissions on all configured directories.
+
+This script is called by the entrypoint to ensure all user-configured
+directories have correct ownership. It reads directory paths from:
+- CONFIG_DIR environment variable
+- Config files in CONFIG_DIR/plugins/
+
+Outputs directory paths that need permission fixing (one per line).
+The entrypoint handles the actual chown operations.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def get_directories_from_config() -> set[str]:
+    """Extract all directory paths from config files."""
+    directories = set()
+
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    plugins_dir = config_dir / "plugins"
+
+    if not plugins_dir.exists():
+        return directories
+
+    # Keys that contain directory paths
+    directory_keys = {
+        # Main destinations
+        "DESTINATION",
+        "DESTINATION_AUDIOBOOK",
+        # Content type routing directories
+        "AA_CONTENT_TYPE_DIR_FICTION",
+        "AA_CONTENT_TYPE_DIR_NON_FICTION",
+        "AA_CONTENT_TYPE_DIR_UNKNOWN",
+        "AA_CONTENT_TYPE_DIR_MAGAZINE",
+        "AA_CONTENT_TYPE_DIR_COMIC",
+        "AA_CONTENT_TYPE_DIR_STANDARDS",
+        "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+        "AA_CONTENT_TYPE_DIR_OTHER",
+        # Legacy keys (in case of old configs)
+        "INGEST_DIR",
+        "INGEST_DIR_AUDIOBOOK",
+        "INGEST_DIR_BOOK_FICTION",
+        "INGEST_DIR_BOOK_NON_FICTION",
+        "INGEST_DIR_BOOK_UNKNOWN",
+        "INGEST_DIR_MAGAZINE",
+        "INGEST_DIR_COMIC_BOOK",
+        "INGEST_DIR_STANDARDS_DOCUMENT",
+        "INGEST_DIR_MUSICAL_SCORE",
+        "INGEST_DIR_OTHER",
+        "LIBRARY_PATH",
+        "LIBRARY_PATH_AUDIOBOOK",
+    }
+
+    # Read all JSON config files
+    for config_file in plugins_dir.glob("*.json"):
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+
+            for key in directory_keys:
+                if key in config:
+                    value = config[key]
+                    if value and isinstance(value, str) and value.startswith("/"):
+                        directories.add(value)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    return directories
+
+
+def main():
+    """Output all configured directories that exist."""
+    directories = get_directories_from_config()
+
+    # Filter to directories that actually exist
+    existing = []
+    for dir_path in directories:
+        path = Path(dir_path)
+        if path.exists() and path.is_dir():
+            existing.append(dir_path)
+
+    # Output one directory per line
+    for dir_path in sorted(existing):
+        print(dir_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_env_docs.py

@@ -0,0 +1,433 @@
+#!/usr/bin/env python3
+"""Generate markdown documentation for environment variables from the settings registry.
+
+This script extracts all settings that support environment variable configuration
+and generates a comprehensive markdown file documenting each option.
+
+Usage:
+    python scripts/generate_env_docs.py [--output path/to/output.md]
+
+The generated documentation includes:
+- Environment variable name
+- Description
+- Type (string, number, boolean, etc.)
+- Default value
+- Organizational grouping by settings tab/group
+"""
+
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Add project root to path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+
+def get_field_type_name(field) -> str:
+    """Get a human-readable type name for a field."""
+    from shelfmark.core.settings_registry import (
+        CheckboxField,
+        MultiSelectField,
+        NumberField,
+        OrderableListField,
+        PasswordField,
+        SelectField,
+        TextField,
+    )
+
+    if isinstance(field, CheckboxField):
+        return "boolean"
+    elif isinstance(field, NumberField):
+        return "number"
+    elif isinstance(field, SelectField):
+        return "string (choice)"
+    elif isinstance(field, MultiSelectField):
+        return "string (comma-separated)"
+    elif isinstance(field, OrderableListField):
+        return "JSON array"
+    elif isinstance(field, PasswordField):
+        return "string (secret)"
+    elif isinstance(field, TextField):
+        return "string"
+    else:
+        return "string"
+
+
+def format_default_value(field) -> str:
+    """Format the default value for display."""
+    default = field.default
+
+    if default is None:
+        return "_none_"
+    elif isinstance(default, bool):
+        return f"`{str(default).lower()}`"
+    elif isinstance(default, (int, float)):
+        return f"`{default}`"
+    elif isinstance(default, str):
+        if default == "":
+            return "_empty string_"
+        return f"`{default}`"
+    elif isinstance(default, list):
+        if not default:
+            return "_empty list_"
+        # For simple lists, show comma-separated values
+        if all(isinstance(item, str) for item in default):
+            return f"`{','.join(default)}`"
+        # For complex lists (e.g., OrderableListField defaults), summarize
+        return f"_see UI for defaults_"
+    else:
+        return f"`{default}`"
+
+
+def get_select_options(field) -> Optional[List[str]]:
+    """Get the available options for a SelectField.
+
+    Returns options formatted as 'value (label)' or just 'value' if they match,
+    so users know the actual values to use in environment variables.
+    """
+    from shelfmark.core.settings_registry import SelectField
+
+    if not isinstance(field, SelectField):
+        return None
+
+    options = field.options
+    if callable(options):
+        try:
+            options = options()
+        except Exception:
+            return None
+
+    if not options:
+        return None
+
+    result = []
+    for opt in options:
+        value = opt.get("value", "")
+        label = opt.get("label", "")
+
+        # Format as "value (label)" unless they're the same or value is empty
+        if value == "":
+            result.append(f'`""` ({label})')
+        elif value == label or not label:
+            result.append(f"`{value}`")
+        else:
+            result.append(f"`{value}` ({label})")
+
+    return result
+
+
+def _generate_bootstrap_env_docs() -> List[str]:
+    """Generate documentation for bootstrap environment variables from env.py."""
+    # These are environment variables defined in env.py that are used before
+    # the settings registry is available
+    bootstrap_vars = [
+        {
+            "name": "CONFIG_DIR",
+            "description": "Directory for storing configuration files and plugin settings.",
+            "type": "string (path)",
+            "default": "/config",
+        },
+        {
+            "name": "LOG_ROOT",
+            "description": "Root directory for log files.",
+            "type": "string (path)",
+            "default": "/var/log/",
+        },
+        {
+            "name": "TMP_DIR",
+            "description": "Staging directory for downloads before moving to destination.",
+            "type": "string (path)",
+            "default": "/tmp/shelfmark",
+        },
+        {
+            "name": "ENABLE_LOGGING",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
+            "type": "boolean",
+            "default": "true",
+        },
+        {
+            "name": "FLASK_HOST",
+            "description": "Host address for the Flask web server.",
+            "type": "string",
+            "default": "0.0.0.0",
+        },
+        {
+            "name": "FLASK_PORT",
+            "description": "Port number for the Flask web server.",
+            "type": "number",
+            "default": "8084",
+        },
+        {
+            "name": "SESSION_COOKIE_SECURE",
+            "description": "Enable secure cookies (requires HTTPS).",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "CWA_DB_PATH",
+            "description": "Path to the Calibre-Web database for authentication integration.",
+            "type": "string (path)",
+            "default": "/auth/app.db",
+        },
+        {
+            "name": "DOCKERMODE",
+            "description": "Indicates the application is running inside a Docker container.",
+            "type": "boolean",
+            "default": "false",
+        },
+    ]
+
+    lines = [
+        "## Bootstrap Configuration",
+        "",
+        "These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
+        "",
+        "| Variable | Description | Type | Default |",
+        "|----------|-------------|------|---------|",
+    ]
+
+    for var in bootstrap_vars:
+        lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
+
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for var in bootstrap_vars:
+        lines.append(f"#### `{var['name']}`")
+        lines.append("")
+        lines.append(var["description"])
+        lines.append("")
+        lines.append(f"- **Type:** {var['type']}")
+        lines.append(f"- **Default:** `{var['default']}`")
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def generate_env_docs() -> str:
+    """Generate markdown documentation for all environment variables."""
+    # Import settings modules to ensure all settings are registered
+    import shelfmark.config.settings  # noqa: F401
+    import shelfmark.release_sources.irc.settings  # noqa: F401
+    import shelfmark.release_sources.prowlarr.settings  # noqa: F401
+    import shelfmark.metadata_providers.hardcover  # noqa: F401
+    import shelfmark.metadata_providers.openlibrary  # noqa: F401
+    import shelfmark.metadata_providers.googlebooks  # noqa: F401
+
+    from shelfmark.core.settings_registry import (
+        ActionButton,
+        HeadingField,
+        get_all_groups,
+        get_all_settings_tabs,
+    )
+
+    tabs = get_all_settings_tabs()
+    groups = {g.name: g for g in get_all_groups()}
+
+    # Organize tabs by group
+    grouped_tabs: Dict[Optional[str], List] = {None: []}
+    for group_name in groups:
+        grouped_tabs[group_name] = []
+
+    for tab in tabs:
+        group_name = tab.group
+        if group_name not in grouped_tabs:
+            grouped_tabs[group_name] = []
+        grouped_tabs[group_name].append(tab)
+
+    # Build markdown output
+    lines = [
+        "# Environment Variables",
+        "",
+        "This document lists all configuration options that can be set via environment variables.",
+        "",
+        "> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
+        "",
+        "## Table of Contents",
+        "",
+    ]
+
+    # Generate TOC
+    toc_entries = [
+        "- [Bootstrap Configuration](#bootstrap-configuration)",
+    ]
+
+    # Ungrouped tabs first
+    for tab in grouped_tabs.get(None, []):
+        anchor = tab.display_name.lower().replace(" ", "-")
+        toc_entries.append(f"- [{tab.display_name}](#{anchor})")
+
+    # Then grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if group_tabs:
+            anchor = group.display_name.lower().replace(" ", "-")
+            toc_entries.append(f"- [{group.display_name}](#{anchor})")
+            for tab in group_tabs:
+                sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
+                toc_entries.append(f"  - [{tab.display_name}](#{sub_anchor})")
+
+    lines.extend(toc_entries)
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Add bootstrap environment variables documentation
+    lines.extend(_generate_bootstrap_env_docs())
+
+    # Generate documentation for ungrouped tabs
+    for tab in grouped_tabs.get(None, []):
+        lines.extend(_generate_tab_docs(tab))
+
+    # Generate documentation for grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if not group_tabs:
+            continue
+
+        lines.append(f"## {group.display_name}")
+        lines.append("")
+
+        for tab in group_tabs:
+            lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
+
+    return "\n".join(lines)
+
+
+def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
+    """Generate documentation for a single settings tab."""
+    from shelfmark.core.settings_registry import ActionButton, HeadingField
+
+    lines = []
+
+    # Section header
+    if group_prefix:
+        lines.append(f"### {group_prefix}: {tab.display_name}")
+        anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
+    else:
+        lines.append(f"## {tab.display_name}")
+
+    lines.append("")
+
+    # Collect env-supported fields
+    env_fields = []
+    for field in tab.fields:
+        # Skip non-value fields
+        if isinstance(field, (ActionButton, HeadingField)):
+            continue
+
+        # Skip fields that don't support ENV vars
+        if not getattr(field, "env_supported", True):
+            continue
+
+        env_fields.append(field)
+
+    if not env_fields:
+        lines.append("_No environment variables for this section._")
+        lines.append("")
+        return lines
+
+    # Generate table
+    lines.append("| Variable | Description | Type | Default |")
+    lines.append("|----------|-------------|------|---------|")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        description = field.description or field.label
+        # Clean up description for table (remove newlines, escape pipes)
+        description = description.replace("\n", " ").replace("|", "\\|").strip()
+
+        field_type = get_field_type_name(field)
+        default = format_default_value(field)
+
+        lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
+
+    lines.append("")
+
+    # Add detailed documentation for each field
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        lines.append(f"#### `{env_var}`")
+        lines.append("")
+        lines.append(f"**{field.label}**")
+        lines.append("")
+
+        if field.description:
+            lines.append(field.description)
+            lines.append("")
+
+        lines.append(f"- **Type:** {get_field_type_name(field)}")
+        lines.append(f"- **Default:** {format_default_value(field)}")
+
+        if getattr(field, "required", False):
+            lines.append("- **Required:** Yes")
+
+        if getattr(field, "requires_restart", False):
+            lines.append("- **Requires restart:** Yes")
+
+        # Show options for SelectField
+        options = get_select_options(field)
+        if options:
+            lines.append(f"- **Options:** {', '.join(options)}")
+
+        # Show constraints for NumberField
+        from shelfmark.core.settings_registry import NumberField
+        if isinstance(field, NumberField):
+            constraints = []
+            if field.min_value is not None:
+                constraints.append(f"min: {field.min_value}")
+            if field.max_value is not None:
+                constraints.append(f"max: {field.max_value}")
+            if constraints:
+                lines.append(f"- **Constraints:** {', '.join(constraints)}")
+
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate markdown documentation for environment variables"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=project_root / "docs" / "environment-variables.md",
+        help="Output file path (default: docs/environment-variables.md)",
+    )
+    parser.add_argument(
+        "--stdout",
+        action="store_true",
+        help="Print to stdout instead of file",
+    )
+    args = parser.parse_args()
+
+    docs = generate_env_docs()
+
+    if args.stdout:
+        print(docs)
+    else:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(docs)
+        print(f"Generated: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_clients.py

@@ -0,0 +1,537 @@
+#!/usr/bin/env python3
+"""
+Test script for download client implementations.
+
+Usage:
+    1. Start the test stack:
+       docker compose -f docker-compose.test-clients.yml up -d
+
+    2. Wait for containers to initialize (first run takes ~30s)
+
+    3. Run this script to verify clients are accessible:
+       python scripts/test_clients.py
+
+    4. Access cwabd at http://localhost:8084
+       - Go to Settings > Prowlarr > Download Clients
+       - Select a client from the dropdown
+       - Click "Test Connection" to verify
+
+Web UIs:
+    - cwabd:        http://localhost:8084
+    - qBittorrent:  http://localhost:8080
+    - Transmission: http://localhost:9091
+    - Deluge:       http://localhost:8112
+    - NZBGet:       http://localhost:6789
+    - SABnzbd:      http://localhost:8085
+    - rTorrent:     http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
+
+Prerequisites (for running this script locally):
+    pip install requests transmission-rpc qbittorrent-api
+
+First-Time Setup:
+    qBittorrent:
+        - Check container logs for temporary password: docker logs test-qbittorrent
+        - Login at http://localhost:8080, change password to something known
+        - Default username is 'admin'
+
+    Transmission:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    Deluge:
+        - Access Web UI at http://localhost:8112 (default password: deluge)
+
+    NZBGet:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    SABnzbd:
+        - Complete the setup wizard at http://localhost:8085
+        - API key will be auto-detected by this script
+        - In cwabd, copy API key from SABnzbd Config > General
+"""
+
+import sys
+import time
+from xmlrpc import client
+
+# Test configuration - matches docker-compose.test-clients.yml
+CONFIG = {
+    # Usenet clients
+    "nzbget": {
+        "url": "http://localhost:6789",
+        "username": "admin",
+        "password": "admin",
+    },
+    "sabnzbd": {
+        "url": "http://localhost:8085",
+        "api_key": None,  # Will be read from config on first run
+    },
+    # Torrent clients
+    "qbittorrent": {
+        "url": "http://localhost:8080",
+        "username": "admin",
+        "password": "5NCngsHXm",  # Temp password from: docker logs test-qbittorrent | grep password
+    },
+    "transmission": {
+        "url": "http://localhost:9091",
+        "username": "admin",
+        "password": "admin",
+    },
+    "deluge": {
+        "url": "http://localhost:8112",
+        "password": "deluge",
+    },
+    "rtorrent": {
+        "url": "http://localhost:8000/RPC2",
+    },
+}
+
+# Test magnet link (Ubuntu ISO - legal, small metadata)
+TEST_MAGNET = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=ubuntu-22.04.3-live-server-amd64.iso"
+
+
+def test_nzbget():
+    """Test NZBGet connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing NZBGet")
+    print("=" * 50)
+
+    url = CONFIG["nzbget"]["url"]
+    username = CONFIG["nzbget"]["username"]
+    password = CONFIG["nzbget"]["password"]
+
+    try:
+        # Test connection via JSON-RPC
+        rpc_url = f"{url}/jsonrpc"
+        response = requests.post(
+            rpc_url,
+            json={"method": "version", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("result", "unknown")
+        print(f"  Connected to NZBGet {version}")
+
+        # Test status
+        response = requests.post(
+            rpc_url,
+            json={"method": "status", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        status = response.json().get("result", {})
+        print(f"  Server state: {'Paused' if status.get('ServerPaused') else 'Running'}")
+        print(f"  Downloads in queue: {status.get('DownloadedSizeMB', 0)} MB downloaded")
+
+        print("  SUCCESS: NZBGet is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to NZBGet")
+        print("  Is the container running? docker ps | grep nzbget")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_sabnzbd():
+    """Test SABnzbd connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing SABnzbd")
+    print("=" * 50)
+
+    url = CONFIG["sabnzbd"]["url"]
+    api_key = CONFIG["sabnzbd"]["api_key"]
+
+    # Try to get API key from config if not set
+    if not api_key:
+        try:
+            import os
+            ini_path = ".local/test-clients/sabnzbd/config/sabnzbd.ini"
+            if os.path.exists(ini_path):
+                with open(ini_path) as f:
+                    for line in f:
+                        if line.startswith("api_key"):
+                            api_key = line.split("=")[1].strip()
+                            print(f"  Found API key in config: {api_key[:8]}...")
+                            break
+        except Exception as e:
+            print(f"  Could not read API key from config: {e}")
+
+    if not api_key:
+        print("  ERROR: No API key configured")
+        print("  Please access http://localhost:8085 and complete initial setup")
+        print("  Then copy the API key from Config > General")
+        return False
+
+    try:
+        # Test connection
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "version", "output": "json"},
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("version", "unknown")
+        print(f"  Connected to SABnzbd {version}")
+
+        # Test queue status
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "queue", "output": "json"},
+            timeout=10,
+        )
+        queue = response.json().get("queue", {})
+        print(f"  Queue status: {queue.get('status', 'unknown')}")
+        print(f"  Items in queue: {len(queue.get('slots', []))}")
+
+        print("  SUCCESS: SABnzbd is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to SABnzbd")
+        print("  Is the container running? docker ps | grep sabnzbd")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_qbittorrent():
+    """Test qBittorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing qBittorrent")
+    print("=" * 50)
+
+    try:
+        import qbittorrentapi
+
+        url = CONFIG["qbittorrent"]["url"]
+        username = CONFIG["qbittorrent"]["username"]
+        password = CONFIG["qbittorrent"]["password"]
+
+        # Parse URL for host/port
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+
+        client = qbittorrentapi.Client(
+            host=parsed.hostname,
+            port=parsed.port or 8080,
+            username=username,
+            password=password,
+        )
+
+        # Test connection
+        client.auth_log_in()
+        version = client.app.version
+        print(f"  Connected to qBittorrent {version}")
+
+        # Get torrent list
+        torrents = client.torrents_info()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        result = client.torrents_add(urls=TEST_MAGNET, is_paused=True)
+        if result == "Ok.":
+            # Wait a moment for it to be added
+            time.sleep(1)
+            torrents = client.torrents_info()
+            if torrents:
+                test_torrent = torrents[-1]  # Most recently added
+                print(f"  Added test torrent: {test_torrent.name[:50]}...")
+                print(f"  Status: {test_torrent.state}")
+
+                # Remove it
+                client.torrents_delete(torrent_hashes=test_torrent.hash, delete_files=True)
+                print("  Removed test torrent")
+        else:
+            print(f"  Add result: {result}")
+
+        print("  SUCCESS: qBittorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: qbittorrent-api not installed")
+        print("  Run: pip install qbittorrent-api")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Forbidden" in str(e) or "401" in str(e):
+            print("\n  Authentication failed. Check password:")
+            print("  1. docker logs test-qbittorrent | grep password")
+            print("  2. Login to http://localhost:8080 and set a known password")
+        return False
+
+
+def test_transmission():
+    """Test Transmission connection."""
+    print("\n" + "=" * 50)
+    print("Testing Transmission")
+    print("=" * 50)
+
+    try:
+        from transmission_rpc import Client
+        from urllib.parse import urlparse
+
+        url = CONFIG["transmission"]["url"]
+        parsed = urlparse(url)
+
+        client = Client(
+            host=parsed.hostname,
+            port=parsed.port or 9091,
+            username=CONFIG["transmission"]["username"],
+            password=CONFIG["transmission"]["password"],
+        )
+
+        # Test connection
+        session = client.get_session()
+        print(f"  Connected to Transmission {session.version}")
+
+        # Get torrent list
+        torrents = client.get_torrents()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent = client.add_torrent(TEST_MAGNET, paused=True)
+        print(f"  Added test torrent: {torrent.name[:50]}...")
+
+        # Get status
+        status = client.get_torrent(torrent.id)
+        print(f"  Status: {status.status} ({status.percent_done * 100:.1f}%)")
+
+        # Remove it
+        client.remove_torrent(torrent.id, delete_data=True)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: Transmission is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: transmission-rpc not installed")
+        print("  Run: pip install transmission-rpc")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_deluge():
+    """Test Deluge Web UI (JSON-RPC) connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing Deluge")
+    print("=" * 50)
+
+    base_url = CONFIG["deluge"]["url"].rstrip("/")
+    password = CONFIG["deluge"]["password"]
+    rpc_url = f"{base_url}/json"
+
+    def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
+        payload = {"id": rpc_id, "method": method, "params": list(params)}
+        resp = session.post(rpc_url, json=payload, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("error"):
+            err = data["error"]
+            if isinstance(err, dict):
+                raise Exception(err.get("message") or str(err))
+            raise Exception(str(err))
+        return data.get("result")
+
+    try:
+        session = requests.Session()
+
+        # Authenticate to Deluge Web
+        if rpc_call(session, 1, "auth.login", password) is not True:
+            raise Exception("Authentication failed (check Deluge Web UI password)")
+
+        # Ensure Deluge Web is connected to a daemon
+        if rpc_call(session, 2, "web.connected") is not True:
+            hosts = rpc_call(session, 3, "web.get_hosts") or []
+            if not hosts:
+                raise Exception(
+                    "Deluge Web UI isn't connected to Deluge core (no hosts configured). "
+                    "Add/connect a daemon in Deluge Web UI → Connection Manager."
+                )
+
+            host_id = hosts[0][0]
+            for entry in hosts:
+                if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
+                    host_id = entry[0]
+                    break
+
+            rpc_call(session, 4, "web.connect", host_id)
+
+            if rpc_call(session, 5, "web.connected") is not True:
+                raise Exception(
+                    "Deluge Web UI couldn't connect to Deluge core. "
+                    "Check Deluge Web UI → Connection Manager."
+                )
+
+        version = rpc_call(session, 6, "daemon.info")
+        print(f"  Connected to Deluge {version}")
+
+        torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
+
+        if torrent_id:
+            torrent_id = str(torrent_id)
+            print(f"  Added test torrent: {torrent_id[:20]}...")
+
+            status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
+            state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
+            progress = status.get("progress", 0) if isinstance(status, dict) else 0
+            print(f"  Status: {state} ({progress:.1f}%)")
+
+            rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
+            print("  Removed test torrent")
+        else:
+            print("  WARNING: Could not add test torrent")
+
+        print("  SUCCESS: Deluge is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to Deluge Web UI")
+        print("  Is the container running? docker ps | grep deluge")
+        return False
+    except requests.exceptions.Timeout:
+        print("  ERROR: Deluge Web UI connection timed out")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "auth" in str(e).lower() or "login" in str(e).lower():
+            print("  Check Deluge Web UI password (default: deluge)")
+        return False
+
+def test_rtorrent():
+    """Test rTorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing rTorrent")
+    print("=" * 50)
+
+    try:
+        import xmlrpc.client
+
+        url = "http://localhost:8000/RPC2"
+        client = xmlrpc.client.ServerProxy(url)
+
+        # Test connection
+        version = client.system.library_version()
+        print(f"  Connected to rTorrent {version}")
+
+        # Get torrent list
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+
+        label = "automated"
+
+        commands = []
+        if label:
+            commands.append(f"d.custom1.set={label}")
+
+        download_dir = "/downloads"
+        if download_dir:
+            commands.append(f"d.directory_base.set={download_dir}")
+
+        # rtorrent is weird in that it doesn't return the torrent ID/hash on add
+        client.load.start("", TEST_MAGNET, ";".join(commands))
+        
+        # but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
+        torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
+        print(f"  Added test torrent: {torrent_id}")
+
+        torrent_list = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal=d.hash=,cat={torrent_id}",
+            "d.hash=",
+            "d.state=",
+            "d.completed_bytes=",
+            "d.size_bytes=",
+            "d.down.rate=",
+            "d.up.rate=",
+            "d.custom1=",
+            "d.complete=",
+        )
+        torrent = torrent_list[0]
+        if not torrent:
+            print("  ERROR: Could not find added torrent in list")
+            return False
+        
+        
+        client.d.erase(torrent_id)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: rTorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: xmlrpc.client not available")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Connection refused" in str(e):
+            print("  Is the container running? docker ps | grep rtorrent")
+        return False
+
+
+def main():
+    print("Download Client Test Suite")
+    print("=" * 50)
+    print("Make sure containers are running:")
+    print("  docker compose -f docker-compose.test-clients.yml up -d")
+
+    results = {}
+
+    # Test usenet clients
+    print("\n" + "=" * 50)
+    print("USENET CLIENTS")
+    print("=" * 50)
+    results["nzbget"] = test_nzbget()
+    results["sabnzbd"] = test_sabnzbd()
+
+    # Test torrent clients
+    print("\n" + "=" * 50)
+    print("TORRENT CLIENTS")
+    print("=" * 50)
+    results["qbittorrent"] = test_qbittorrent()
+    results["transmission"] = test_transmission()
+    results["deluge"] = test_deluge()
+    results["rtorrent"] = test_rtorrent()
+
+    # Summary
+    print("\n" + "=" * 50)
+    print("SUMMARY")
+    print("=" * 50)
+
+    for client, success in results.items():
+        status = "PASS" if success else "FAIL"
+        print(f"  {client}: {status}")
+
+    passed = sum(results.values())
+    total = len(results)
+    print(f"\n  Total: {passed}/{total} passed")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

shelfmark/__init__.py

@@ -0,0 +1 @@
+"""Shelfmark - book search and download service."""

shelfmark/__main__.py

@@ -0,0 +1,8 @@
+"""Package entry point for `python -m shelfmark`."""
+
+from shelfmark.main import app, socketio
+from shelfmark.config.env import FLASK_HOST, FLASK_PORT
+from shelfmark.core.config import config
+
+if __name__ == "__main__":
+    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

shelfmark/api/websocket.py

@@ -28,29 +28,17 @@ class WebSocketManager:
         logger.info("WebSocket manager initialized")
 
     def register_on_first_connect(self, callback: Callable[[], None]):
-        """Register a callback to be called when the first client connects.
-
-        This is useful for warming up resources (like the Cloudflare bypasser)
-        when a user starts using the web UI.
-        """
+        """Register a callback for when the first client connects."""
         self._on_first_connect_callbacks.append(callback)
         logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
 
     def register_on_all_disconnect(self, callback: Callable[[], None]):
-        """Register a callback to be called when all clients disconnect.
-
-        This can be used to trigger cleanup or resource release.
-        """
+        """Register a callback for when all clients disconnect."""
         self._on_all_disconnect_callbacks.append(callback)
         logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
 
     def request_warmup_on_next_connect(self):
-        """Request that warmup callbacks be triggered on the next client connect.
-
-        This is used when resources (like the Cloudflare bypasser) shut down due to
-        inactivity while clients are still connected. The next connect event should
-        trigger warmup even though it's not technically the "first" connection.
-        """
+        """Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
         with self._connection_lock:
             self._needs_rewarm = True
             logger.debug("Warmup requested for next client connect")
@@ -157,6 +145,30 @@ class WebSocketManager:
         except Exception as e:
             logger.error(f"Error broadcasting notification: {e}")
 
+    def broadcast_search_status(
+        self,
+        source: str,
+        provider: str,
+        book_id: str,
+        message: str,
+        phase: str = 'searching'
+    ):
+        """Broadcast search status update for a release source search."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'source': source,
+                'provider': provider,
+                'book_id': book_id,
+                'message': message,
+                'phase': phase,
+            }
+            self.socketio.emit('search_status', data)
+        except Exception as e:
+            logger.error(f"Error broadcasting search status: {e}")
+
 
 # Global WebSocket manager instance
 ws_manager = WebSocketManager()

shelfmark/bypass/__init__.py

@@ -0,0 +1,5 @@
+"""Cloudflare bypass utilities."""
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""

shelfmark/bypass/external_bypasser.py

@@ -0,0 +1,128 @@
+"""External Cloudflare bypasser using FlareSolverr."""
+
+import random
+import time
+from threading import Event
+from typing import TYPE_CHECKING, Optional
+
+import requests
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.utils import normalize_http_url
+
+if TYPE_CHECKING:
+    from shelfmark.download import network
+
+logger = setup_logger(__name__)
+
+# Timeout constants (seconds)
+CONNECT_TIMEOUT = 10
+MAX_READ_TIMEOUT = 120
+READ_TIMEOUT_BUFFER = 15
+
+# Retry settings
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
+
+
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service. Returns HTML or None."""
+    raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
+    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
+    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
+
+    bypasser_url = normalize_http_url(raw_bypasser_url)
+    if not bypasser_url or not bypasser_path:
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
+        return None
+
+    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+
+    try:
+        response = requests.post(
+            f"{bypasser_url}{bypasser_path}",
+            headers={"Content-Type": "application/json"},
+            json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
+            timeout=(CONNECT_TIMEOUT, read_timeout)
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+
+        solution = result.get('solution')
+        html = solution.get('response', '') if solution else ''
+
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+
+        return html
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+
+    return None
+
+
+def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
+    """Check if operation was cancelled and raise exception if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(f"External bypasser cancelled {context}")
+        raise BypassCancelledException("Bypass cancelled")
+
+
+def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
+    """Sleep for the specified duration, checking for cancellation each second."""
+    for _ in range(int(seconds)):
+        _check_cancelled(cancel_flag, "during backoff")
+        time.sleep(1)
+    remaining = seconds - int(seconds)
+    if remaining > 0:
+        time.sleep(remaining)
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional["network.AAMirrorSelector"] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML via external bypasser with retries and mirror rotation."""
+    from shelfmark.download import network as network_module
+
+    sel = selector or network_module.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        _check_cancelled(cancel_flag, "by user")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        _sleep_with_cancellation(delay, cancel_flag)
+
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

shelfmark/bypass/fingerprint.py

@@ -0,0 +1,57 @@
+"""Browser fingerprint profile management for bypass stealth."""
+
+import random
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+COMMON_RESOLUTIONS = [
+    (1920, 1080, 0.35),  
+    (1366, 768, 0.18),   
+    (1536, 864, 0.10),   
+    (1440, 900, 0.08),   
+    (1280, 720, 0.07),   
+    (1600, 900, 0.06),   
+    (1280, 800, 0.05),   
+    (2560, 1440, 0.04),  
+    (1680, 1050, 0.04),  
+    (1920, 1200, 0.03),  
+]
+
+# Current screen size (module-level singleton)
+_current_screen_size: Optional[tuple[int, int]] = None
+
+
+def get_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    if _current_screen_size is None:
+        _current_screen_size = _generate_screen_size()
+        logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
+    return _current_screen_size
+
+
+def rotate_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    old_size = _current_screen_size
+    _current_screen_size = _generate_screen_size()
+    width, height = _current_screen_size
+
+    if old_size:
+        logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
+    else:
+        logger.info(f"Generated screen size: {width}x{height}")
+
+    return _current_screen_size
+
+
+def clear_screen_size() -> None:
+    global _current_screen_size
+    _current_screen_size = None
+
+
+def _generate_screen_size() -> tuple[int, int]:
+    resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
+    weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
+    return random.choices(resolutions, weights=weights)[0]

shelfmark/config/booklore_settings.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.outputs.booklore import (
+    BookloreConfig,
+    BookloreError,
+    booklore_list_libraries,
+    booklore_login,
+)
+
+logger = setup_logger(__name__)
+
+_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
+    "key": None,
+    "library_options": [],
+    "path_options": [],
+}
+
+
+def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
+    return f"{base_url}|{username}|{hash(password)}"
+
+
+def _get_booklore_select_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    # library_id/path_id are not used for login/library listing
+    booklore_config = BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=1,
+        path_id=1,
+        verify_tls=True,
+        refresh_after_upload=True,
+    )
+
+    token = booklore_login(booklore_config)
+    libraries = booklore_list_libraries(booklore_config, token) or []
+    logger.debug("Booklore libraries response: %s", libraries)
+
+    library_options: list[dict[str, Any]] = []
+    path_options: list[dict[str, Any]] = []
+
+    for library in libraries:
+        if not isinstance(library, dict):
+            continue
+
+        library_id = library.get("id")
+        if library_id is None:
+            continue
+
+        library_name = str(library.get("name") or f"Library {library_id}")
+        library_id_str = str(library_id)
+
+        library_options.append({"value": library_id_str, "label": library_name})
+
+        paths = library.get("paths") or []
+        if not isinstance(paths, list):
+            continue
+
+        for path in paths:
+            if not isinstance(path, dict):
+                continue
+
+            path_id = path.get("id")
+            if path_id is None:
+                continue
+
+            path_label = str(path.get("path") or f"Path {path_id}")
+            path_options.append(
+                {
+                    "value": str(path_id),
+                    "label": f"{library_name}: {path_label}",
+                    "childOf": library_id_str,
+                }
+            )
+
+    logger.debug(
+        "Booklore options built: libraries=%d paths=%d",
+        len(library_options),
+        len(path_options),
+    )
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    _BOOKLORE_OPTIONS_CACHE.update(
+        {
+            "key": cache_key,
+            "library_options": library_options,
+            "path_options": path_options,
+        }
+    )
+
+    return library_options, path_options
+
+
+def _get_booklore_cached_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+        return (
+            _BOOKLORE_OPTIONS_CACHE.get("library_options", []),
+            _BOOKLORE_OPTIONS_CACHE.get("path_options", []),
+        )
+
+    return _get_booklore_select_options(base_url, username, password)
+
+
+def get_booklore_library_options() -> list[dict[str, Any]]:
+    """Build Booklore library options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        library_options, _ = _get_booklore_cached_options(base_url, username, password)
+        return library_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore libraries: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
+        return []
+
+
+def get_booklore_path_options() -> list[dict[str, Any]]:
+    """Build Booklore path options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        _, path_options = _get_booklore_cached_options(base_url, username, password)
+        return path_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore paths: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
+        return []
+
+
+def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test the Booklore connection using current form values."""
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
+    password = _get_value("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        return {"success": False, "message": "Booklore URL is required"}
+    if not username:
+        return {"success": False, "message": "Booklore username is required"}
+    if not password:
+        return {"success": False, "message": "Booklore password is required"}
+
+    try:
+        library_options, _ = _get_booklore_select_options(base_url, username, password)
+
+        message = "Connected to Booklore"
+        if library_options:
+            message = f"Connected to Booklore ({len(library_options)} libraries)"
+
+        return {"success": True, "message": message}
+    except BookloreError as exc:
+        return {"success": False, "message": str(exc)}

shelfmark/config/env.py

@@ -0,0 +1,153 @@
+"""Bootstrap environment variables. No local dependencies - import first."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+
+def string_to_bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ["true", "yes", "1", "y"]
+
+
+def _read_debug_from_config() -> bool:
+    """Read DEBUG from env var or config file (import-time safe)."""
+    env_debug = os.environ.get("DEBUG")
+    if env_debug is not None:
+        return string_to_bool(env_debug)
+
+    # Try to read from config file
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    config_file = config_dir / "plugins" / "advanced.json"
+
+    if config_file.exists():
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+                if "DEBUG" in config:
+                    return bool(config["DEBUG"])
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return False
+
+
+def _is_sqlite_file(path: Path) -> bool:
+    """Check if a file is a valid SQLite database by reading magic bytes."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(16)
+            return header[:16] == b"SQLite format 3\x00"
+    except (OSError, PermissionError):
+        return False
+
+
+def _resolve_cwa_db_path() -> Path | None:
+    """Resolve CWA database path from env var or default location."""
+    env_path = os.getenv("CWA_DB_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file() and _is_sqlite_file(path):
+            return path
+
+    # Check default mount path
+    default_path = Path("/auth/app.db")
+    if default_path.exists() and default_path.is_file() and _is_sqlite_file(default_path):
+        return default_path
+
+    return None
+
+
+def _is_config_dir_writable() -> bool:
+    """Check if the config directory exists and is writable."""
+    try:
+        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
+            return False
+        test_file = CONFIG_DIR / ".write_test"
+        test_file.touch()
+        test_file.unlink()
+        return True
+    except (OSError, PermissionError):
+        return False
+
+
+def is_covers_cache_enabled() -> bool:
+    """Check if cover caching is enabled (requires setting + writable config dir)."""
+    from shelfmark.core.config import config
+    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
+    return setting_enabled and _is_config_dir_writable()
+
+
+# =============================================================================
+# Bootstrap paths - needed before settings registry is available
+# =============================================================================
+
+CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
+LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
+LOG_DIR = LOG_ROOT / "shelfmark"
+LOG_FILE = LOG_DIR / "shelfmark.log"
+TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
+INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
+
+
+# =============================================================================
+# Logger configuration - needed before settings registry is available
+# =============================================================================
+
+DEBUG = _read_debug_from_config()
+LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
+ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+
+
+# =============================================================================
+# Flask configuration - needed before app starts
+# =============================================================================
+
+FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
+
+
+# =============================================================================
+# Authentication
+# =============================================================================
+
+SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
+CWA_DB_PATH = _resolve_cwa_db_path()
+
+
+# =============================================================================
+# Version information from Docker build
+# =============================================================================
+
+BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
+RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
+
+
+# =============================================================================
+# Capability detection - runtime checks, not user-configurable
+# =============================================================================
+
+DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
+TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
+USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
+
+
+# =============================================================================
+# Debug/development settings
+# =============================================================================
+
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
+
+
+# =============================================================================
+# Legacy migration support - will be removed in future version
+# =============================================================================
+
+# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
+# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
+_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))

shelfmark/config/security.py

@@ -0,0 +1,290 @@
+"""Authentication settings registration."""
+
+from typing import Any, Dict
+
+from werkzeug.security import generate_password_hash
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    register_settings,
+    register_on_save,
+    load_config_file,
+    TextField,
+    SelectField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+
+logger = setup_logger(__name__)
+
+
+def _migrate_security_settings() -> None:
+    import json
+    from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
+
+    try:
+        config = load_config_file("security")
+        migrated = False
+        
+        # Migrate USE_CWA_AUTH to AUTH_METHOD
+        if "USE_CWA_AUTH" in config:
+            old_value = config.pop("USE_CWA_AUTH")
+            
+            # Only set AUTH_METHOD if it doesn't already exist
+            if "AUTH_METHOD" not in config:
+                if old_value:
+                    config["AUTH_METHOD"] = "cwa"
+                    logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
+                else:
+                    # If USE_CWA_AUTH was False, determine auth method from credentials
+                    if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                        config["AUTH_METHOD"] = "builtin"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
+                    else:
+                        config["AUTH_METHOD"] = "none"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
+                migrated = True
+            else:
+                logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
+                migrated = True
+        
+        # Migrate RESTRICT_SETTINGS_TO_ADMIN to CWA_RESTRICT_SETTINGS_TO_ADMIN
+        if "RESTRICT_SETTINGS_TO_ADMIN" in config:
+            old_value = config.pop("RESTRICT_SETTINGS_TO_ADMIN")
+            
+            # Only migrate if new key doesn't exist
+            if "CWA_RESTRICT_SETTINGS_TO_ADMIN" not in config:
+                config["CWA_RESTRICT_SETTINGS_TO_ADMIN"] = old_value
+                logger.info(f"Migrated RESTRICT_SETTINGS_TO_ADMIN={old_value} to CWA_RESTRICT_SETTINGS_TO_ADMIN={old_value}")
+                migrated = True
+            else:
+                logger.info("Removed deprecated RESTRICT_SETTINGS_TO_ADMIN setting (CWA_RESTRICT_SETTINGS_TO_ADMIN already exists)")
+                migrated = True
+        
+        # Save config if any migrations occurred
+        if migrated:
+            _ensure_config_dir("security")
+            config_path = _get_config_file_path("security")
+            with open(config_path, 'w') as f:
+                json.dump(config, f, indent=2)
+            logger.info("Security settings migration completed successfully")
+        else:
+            logger.debug("No security settings migration needed")
+    
+    except FileNotFoundError:
+        logger.debug("No existing security config file found - nothing to migrate")
+    except Exception as e:
+        logger.error(f"Failed to migrate security settings: {e}")
+
+
+def _clear_builtin_credentials() -> Dict[str, Any]:
+    """Clear built-in credentials to allow public access."""
+    import json
+    from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
+
+    try:
+        config = load_config_file("security")
+        config.pop("BUILTIN_USERNAME", None)
+        config.pop("BUILTIN_PASSWORD_HASH", None)
+
+        _ensure_config_dir("security")
+        config_path = _get_config_file_path("security")
+        with open(config_path, 'w') as f:
+            json.dump(config, f, indent=2)
+
+        logger.info("Cleared credentials")
+        return {"success": True, "message": "Credentials cleared. The app is now publicly accessible."}
+
+    except Exception as e:
+        logger.error(f"Failed to clear credentials: {e}")
+        return {"success": False, "message": f"Failed to clear credentials: {str(e)}"}
+
+
+def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Custom save handler for security settings.
+
+    Handles password validation and hashing:
+    - If new password is provided, validate confirmation and hash it
+    - If password fields are empty, preserve existing hash
+    - Never store raw passwords
+    - Ensure username is present if password is set
+
+    Returns:
+        Dict with processed values to save and any validation errors.
+    """
+    password = values.get("BUILTIN_PASSWORD", "")
+    password_confirm = values.get("BUILTIN_PASSWORD_CONFIRM", "")
+
+    # Remove raw password fields - they should never be persisted
+    values.pop("BUILTIN_PASSWORD", None)
+    values.pop("BUILTIN_PASSWORD_CONFIRM", None)
+
+    # If password is provided, validate and hash it
+    if password:
+        if not values.get("BUILTIN_USERNAME"):
+            return {
+                "error": True,
+                "message": "Username cannot be empty",
+                "values": values
+            }
+
+        if password != password_confirm:
+            return {
+                "error": True,
+                "message": "Passwords do not match",
+                "values": values
+            }
+
+        if len(password) < 4:
+            return {
+                "error": True,
+                "message": "Password must be at least 4 characters",
+                "values": values
+            }
+
+        # Hash the password
+        values["BUILTIN_PASSWORD_HASH"] = generate_password_hash(password)
+        logger.info("Password hash updated")
+
+    # If no password provided but username is being set, preserve existing hash
+    elif "BUILTIN_USERNAME" in values:
+        existing = load_config_file("security")
+        if "BUILTIN_PASSWORD_HASH" in existing:
+            values["BUILTIN_PASSWORD_HASH"] = existing["BUILTIN_PASSWORD_HASH"]
+
+    return {"error": False, "values": values}
+
+
+@register_settings("security", "Security", icon="shield", order=5)
+def security_settings():  
+    """Security and authentication settings."""
+    from shelfmark.config.env import CWA_DB_PATH
+
+    cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
+
+    auth_method_options = [
+        {"label": "No Authentication", "value": "none"},
+        {"label": "Username/Password", "value": "builtin"},
+        {"label": "Proxy Authentication", "value": "proxy"},
+    ]
+    if cwa_db_available:
+        auth_method_options.append({"label": "Calibre-Web Database", "value": "cwa"})
+
+    auth_method_description = "Select the authentication method for accessing Shelfmark."
+    if not cwa_db_available:
+        auth_method_description += " Calibre-Web database option requires mounting your Calibre-Web app.db to /auth/app.db."
+
+    fields = [
+        SelectField(
+            key="AUTH_METHOD",
+            label="Authentication Method",
+            description=auth_method_description,
+            options=auth_method_options,
+            default="none",
+            env_supported=False,
+        ),
+        TextField(
+            key="BUILTIN_USERNAME",
+            label="Username",
+            description="Set a username and password to require login. Leave both empty for public access.",
+            placeholder="Enter username",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        PasswordField(
+            key="BUILTIN_PASSWORD",
+            label="Set Password",
+            description="Fill in to set or change the password.",
+            placeholder="Enter new password",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        PasswordField(
+            key="BUILTIN_PASSWORD_CONFIRM",
+            label="Confirm Password",
+            placeholder="Confirm new password",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        ActionButton(
+            key="clear_credentials",
+            label="Clear Credentials",
+            description="Remove login requirement and make the app publicly accessible.",
+            style="danger",
+            callback=_clear_builtin_credentials,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        TextField(
+            key="PROXY_AUTH_USER_HEADER",
+            label="Proxy Auth User Header",
+            description=(
+                "The HTTP header your proxy uses to pass the authenticated username."
+            ),
+            placeholder="e.g. X-Auth-User",
+            default="X-Auth-User",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        TextField(
+            key="PROXY_AUTH_LOGOUT_URL",
+            label="Proxy Auth Logout URL",
+            description=(
+                "The URL to redirect users to for logging out."
+                " Leave empty to disable logout functionality."
+            ),
+            placeholder="https://myauth.example.com/logout",
+            default="",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        CheckboxField(
+            key="PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
+            label="Restrict Settings to Admins authenticated via Proxy",
+            description=(
+                "Only users in the admin group can access settings."
+            ),
+            default=False,
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        TextField(
+            key="PROXY_AUTH_ADMIN_GROUP_HEADER",
+            label="Proxy Auth Admin Group Header",
+            description=(
+                "The HTTP header your proxy uses to pass the user's groups/roles."
+            ),
+            placeholder="e.g. X-Auth-Groups",
+            default="X-Auth-Groups",
+            env_supported=False,
+            show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
+        ),
+        TextField(
+            key="PROXY_AUTH_ADMIN_GROUP_NAME",
+            label="Proxy Auth Admin Group Name",
+            description=(
+                "The name of the group/role that should have admin access."
+            ),
+            placeholder="e.g. admins",
+            default="admins",
+            env_supported=False,
+            show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
+        ),
+        CheckboxField(
+            key="CWA_RESTRICT_SETTINGS_TO_ADMIN",
+            label="Restrict Settings to Admins authenticated via Calibre-Web",
+            description=(
+                "Only users with admin role in Calibre-Web can access settings."
+            ),
+            default=False,
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "cwa"},
+        ),
+    ]
+
+    return fields
+
+
+# Register the on_save handler for this tab
+register_on_save("security", _on_save_security)

shelfmark/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core module - shared models, queue, and utilities."""
+
+from shelfmark.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
+from shelfmark.core.queue import BookQueue, book_queue
+from shelfmark.core.logger import setup_logger

shelfmark/core/cache.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 from functools import wraps
 from typing import Any, Callable, Dict, Optional, TypeVar
 
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -24,24 +24,13 @@ class CacheService:
     """Thread-safe in-memory cache with TTL support."""
 
     def __init__(self, max_size: int = 1000):
-        """Initialize cache service.
-
-        Args:
-            max_size: Maximum number of entries before oldest are evicted.
-        """
+        """Initialize cache with max_size entries before eviction."""
         self._cache: Dict[str, CacheEntry] = {}
         self._lock = threading.Lock()
         self._max_size = max_size
 
     def get(self, key: str) -> Optional[Any]:
-        """Get cached value if not expired.
-
-        Args:
-            key: Cache key to retrieve.
-
-        Returns:
-            Cached value or None if not found/expired.
-        """
+        """Get cached value if not expired."""
         with self._lock:
             entry = self._cache.get(key)
             if entry is None:
@@ -54,13 +43,7 @@ class CacheService:
             return entry.value
 
     def set(self, key: str, value: Any, ttl: int) -> None:
-        """Cache value with TTL.
-
-        Args:
-            key: Cache key.
-            value: Value to cache.
-            ttl: Time to live in seconds.
-        """
+        """Cache value with TTL in seconds."""
         with self._lock:
             # Evict oldest entries if at capacity
             if len(self._cache) >= self._max_size:
@@ -72,14 +55,7 @@ class CacheService:
             )
 
     def invalidate(self, key: str) -> bool:
-        """Remove specific cache entry.
-
-        Args:
-            key: Cache key to remove.
-
-        Returns:
-            True if entry was removed, False if not found.
-        """
+        """Remove specific cache entry. Returns True if found."""
         with self._lock:
             if key in self._cache:
                 del self._cache[key]
@@ -92,11 +68,7 @@ class CacheService:
             self._cache.clear()
 
     def cleanup_expired(self) -> int:
-        """Remove all expired entries.
-
-        Returns:
-            Number of entries removed.
-        """
+        """Remove all expired entries. Returns count removed."""
         with self._lock:
             now = time.time()
             expired_keys = [
@@ -108,10 +80,7 @@ class CacheService:
             return len(expired_keys)
 
     def _evict_oldest(self) -> None:
-        """Evict oldest entries (by expiration time) to make room.
-
-        Called with lock held.
-        """
+        """Evict ~10% of oldest entries. Called with lock held."""
         if not self._cache:
             return
 
@@ -126,11 +95,7 @@ class CacheService:
             del self._cache[key]
 
     def stats(self) -> Dict[str, int]:
-        """Get cache statistics.
-
-        Returns:
-            Dict with size and max_size.
-        """
+        """Get cache statistics (size, max_size)."""
         with self._lock:
             return {
                 "size": len(self._cache),
@@ -148,15 +113,7 @@ def get_metadata_cache() -> CacheService:
 
 
 def cache_key(*args, **kwargs) -> str:
-    """Generate cache key from arguments.
-
-    Args:
-        *args: Positional arguments to include in key.
-        **kwargs: Keyword arguments to include in key.
-
-    Returns:
-        String cache key.
-    """
+    """Generate cache key from arguments."""
     parts = [str(arg) for arg in args]
     parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
     return ":".join(parts)
@@ -168,23 +125,12 @@ def cacheable(
     ttl_default: int = 300,
     key_prefix: str = ""
 ):
-    """Decorator for caching function results.
-
-    Args:
-        ttl: Static time to live in seconds (use this OR ttl_key, not both).
-        ttl_key: Config key to read TTL from (e.g., "METADATA_CACHE_SEARCH_TTL").
-        ttl_default: Default TTL if ttl_key not found in config.
-        key_prefix: Optional prefix for cache keys.
-
-    Examples:
-        @cacheable(ttl=300, key_prefix="hardcover:search")  # Static TTL
-        @cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", key_prefix="hardcover:search")  # Dynamic TTL
-    """
+    """Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
     def decorator(func: Callable[..., T]) -> Callable[..., T]:
         @wraps(func)
         def wrapper(*args, **kwargs) -> T:
             # Check if metadata caching is enabled
-            from cwa_book_downloader.core.config import config
+            from shelfmark.core.config import config
 
             if not config.get("METADATA_CACHE_ENABLED", True):
                 # Caching disabled, execute function directly
@@ -211,11 +157,9 @@ def cacheable(
             # Check cache
             cached = _metadata_cache.get(key)
             if cached is not None:
-                logger.debug(f"Cache hit: {key}")
                 return cached
 
             # Execute function and cache result
-            logger.debug(f"Cache miss: {key}")
             result = func(*args, **kwargs)
 
             # Only cache non-None results

shelfmark/core/config.py

@@ -12,7 +12,7 @@ def _get_registry():
     """Lazy import of settings registry to avoid circular imports."""
     global _registry_module
     if _registry_module is None:
-        from cwa_book_downloader.core import settings_registry
+        from shelfmark.core import settings_registry
         _registry_module = settings_registry
     return _registry_module
 
@@ -21,7 +21,7 @@ def _get_env():
     """Lazy import of env module for fallback values."""
     global _env_module
     if _env_module is None:
-        from cwa_book_downloader.config import env
+        from shelfmark.config import env
         _env_module = env
     return _env_module
 
@@ -65,11 +65,12 @@ class Config:
 
     def _load_settings(self) -> None:
         """Load all settings from the registry."""
-        # Ensure all plugin settings are registered before loading
-        # This handles cases where config is accessed before plugins are imported
+        # Ensure all settings modules are imported before loading
+        # This handles cases where config is accessed before settings are registered
         try:
-            import cwa_book_downloader.release_sources  # noqa: F401
-            import cwa_book_downloader.metadata_providers  # noqa: F401
+            import shelfmark.config.settings  # noqa: F401 - main app settings
+            import shelfmark.release_sources  # noqa: F401 - plugin settings
+            import shelfmark.metadata_providers  # noqa: F401 - plugin settings
         except ImportError:
             pass
 

shelfmark/core/image_cache.py

@@ -10,7 +10,7 @@ from typing import Any, Dict, Optional, Tuple
 
 import requests
 
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -92,10 +92,13 @@ class ImageCacheService:
 
     def _load_index(self) -> None:
         """Load cache index from disk."""
+        if not self.index_path.exists():
+            self._index = {}
+            return
+
         try:
-            if self.index_path.exists():
-                with open(self.index_path, 'r') as f:
-                    self._index = json.load(f)
+            with open(self.index_path, 'r') as f:
+                self._index = json.load(f)
         except (json.JSONDecodeError, IOError):
             self._index = {}
 
@@ -179,9 +182,7 @@ class ImageCacheService:
         """Check if a cache entry is expired."""
         if self.ttl_seconds == 0:
             return False
-
-        cached_at = entry.get('cached_at', 0)
-        return (time.time() - cached_at) > self.ttl_seconds
+        return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
 
     def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
         """Check if a negative cache entry is expired.
@@ -193,12 +194,8 @@ class ImageCacheService:
             return False
 
         cached_at = entry.get('cached_at', 0)
-
-        # Transient failures (timeouts, connection errors) use shorter TTL
-        if entry.get('transient', False):
-            return (time.time() - cached_at) > TRANSIENT_CACHE_TTL
-
-        return (time.time() - cached_at) > NEGATIVE_CACHE_TTL
+        ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
+        return (time.time() - cached_at) > ttl
 
     def _calculate_total_size(self) -> int:
         """Calculate total size of cached images."""
@@ -255,14 +252,13 @@ class ImageCacheService:
         with self._lock:
             entry = self._index.get(cache_id)
 
+            # Try reloading from disk if not found (handles multiprocess case)
             if not entry:
-                # Try reloading from disk (handles multiprocess case)
                 self._load_index()
                 entry = self._index.get(cache_id)
-
-            if not entry:
-                self._misses += 1
-                return None
+                if not entry:
+                    self._misses += 1
+                    return None
 
             # Check for negative cache (failed fetch)
             if entry.get('negative', False):
@@ -526,10 +522,8 @@ class ImageCacheService:
             self.put_negative(cache_id, transient=True)
             return None
         except requests.exceptions.HTTPError as e:
-            if e.response is not None and e.response.status_code == 404:
-                self.put_negative(cache_id)
-            else:
-                self.put_negative(cache_id, transient=True)
+            is_404 = e.response is not None and e.response.status_code == 404
+            self.put_negative(cache_id, transient=not is_404)
             return None
         except Exception:
             return None
@@ -550,8 +544,8 @@ def get_image_cache() -> ImageCacheService:
     if _instance is None:
         with _instance_lock:
             if _instance is None:
-                from cwa_book_downloader.core.config import config
-                from cwa_book_downloader.config.env import CONFIG_DIR
+                from shelfmark.core.config import config
+                from shelfmark.config.env import CONFIG_DIR
 
                 cache_dir = CONFIG_DIR / "covers"
                 max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
@@ -563,7 +557,7 @@ def get_image_cache() -> ImageCacheService:
                     max_size_mb=max_size_mb,
                     ttl_seconds=ttl_seconds,
                 )
-                logger.info(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
+                logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
 
     return _instance
 

shelfmark/core/logger.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from logging.handlers import RotatingFileHandler
 from typing import Any
 
-from cwa_book_downloader.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
+from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
 
 
 class CustomLogger(logging.Logger):
@@ -40,11 +40,21 @@ class CustomLogger(logging.Logger):
 
     def log_resource_usage(self):
         import psutil
+
+        # Sum RSS of all processes for actual app memory
+        app_memory_mb = 0
+        for proc in psutil.process_iter(['memory_info']):
+            try:
+                if proc.info['memory_info']:
+                    app_memory_mb += proc.info['memory_info'].rss / (1024 * 1024)
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+
         memory = psutil.virtual_memory()
+        system_used_mb = memory.used / (1024 * 1024)
         available_mb = memory.available / (1024 * 1024)
-        memory_used_mb = memory.used / (1024 * 1024)
         cpu_percent = psutil.cpu_percent()
-        self.debug(f"Container Memory: Available={available_mb:.2f} MB, Used={memory_used_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
+        self.debug(f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
 
 
 def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
@@ -62,17 +72,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
 
     # Create logger as CustomLogger instance
     logger = CustomLogger(name)
-    log_level = logging.INFO
-    if LOG_LEVEL == "DEBUG":
-        log_level = logging.DEBUG
-    elif LOG_LEVEL == "INFO":
-        log_level = logging.INFO
-    elif LOG_LEVEL == "WARNING":
-        log_level = logging.WARNING
-    elif LOG_LEVEL == "ERROR":
-        log_level = logging.ERROR
-    elif LOG_LEVEL == "CRITICAL":
-        log_level = logging.CRITICAL
+    log_level = getattr(logging, LOG_LEVEL, logging.INFO)
     logger.setLevel(log_level)
 
     formatter = logging.Formatter(

shelfmark/core/mirrors.py

@@ -0,0 +1,231 @@
+"""Centralized mirror configuration for all download sources."""
+
+from typing import List
+
+from shelfmark.core.utils import normalize_http_url
+
+# Lazy import to avoid circular imports
+_config_module = None
+
+
+def _get_config():
+    """Lazy import of config module to avoid circular imports."""
+    global _config_module
+    if _config_module is None:
+        from shelfmark.core.config import config
+        _config_module = config
+    return _config_module
+
+
+# Default mirror lists (hardcoded fallbacks)
+DEFAULT_AA_MIRRORS = [
+    "https://annas-archive.se",
+    "https://annas-archive.li",
+    "https://annas-archive.pm",
+    "https://annas-archive.in",
+]
+
+DEFAULT_LIBGEN_MIRRORS = [
+    "https://libgen.gl",
+    "https://libgen.li",
+    "https://libgen.bz",
+    "https://libgen.la",
+    "https://libgen.vg",
+]
+
+DEFAULT_ZLIB_MIRRORS = [
+    "https://z-lib.fm",
+    "https://z-lib.gs",
+    "https://z-lib.id",
+    "https://z-library.sk",
+    "https://zlibrary-global.se",
+]
+
+DEFAULT_WELIB_MIRRORS = [
+    "https://welib.org",
+]
+
+
+def _normalize_mirror_url(url: str) -> str:
+    return normalize_http_url(url, default_scheme="https")
+
+
+def get_aa_mirrors() -> List[str]:
+    """
+    Get Anna's Archive mirrors from config + defaults.
+
+    Returns:
+        List of AA mirror URLs, starting with defaults then custom additions.
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("AA_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_libgen_mirrors() -> List[str]:
+    """
+    Get LibGen mirrors: defaults + any additional from config.
+
+    Returns:
+        List of LibGen mirror URLs (defaults first, then custom additions).
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_mirrors() -> List[str]:
+    """
+    Get Z-Library mirrors, with primary first.
+
+    Returns:
+        List of Z-Library mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_primary_url() -> str:
+    """
+    Get the primary Z-Library mirror URL.
+
+    Returns:
+        Primary Z-Library mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+
+
+def get_zlib_url_template() -> str:
+    """
+    Get Z-Library URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_zlib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_welib_mirrors() -> List[str]:
+    """
+    Get Welib mirrors, with primary first.
+
+    Returns:
+        List of Welib mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_WELIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("WELIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_welib_primary_url() -> str:
+    """
+    Get the primary Welib mirror URL.
+
+    Returns:
+        Primary Welib mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+
+
+def get_welib_url_template() -> str:
+    """
+    Get Welib URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_welib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_zlib_cookie_domains() -> set:
+    """
+    Get set of Z-Library domains that need full cookie handling.
+
+    Used by internal_bypasser for CF bypass cookie management.
+
+    Returns:
+        Set of domain strings (without protocol).
+    """
+    domains = set()
+
+    # Add all default domains
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized:
+            domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+            domains.add(domain)
+
+    # Add custom domains
+    config = _get_config()
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized:
+                domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+                domains.add(domain)
+
+    return domains

shelfmark/core/models.py

@@ -14,17 +14,6 @@ def build_filename(
     year: Optional[str] = None,
     fmt: Optional[str] = None,
 ) -> str:
-    """Build sanitized filename: 'Author - Title (Year).format'
-
-    Args:
-        title: Book title (required)
-        author: Book author
-        year: Publication year
-        fmt: File format/extension
-
-    Returns:
-        Sanitized filename safe for filesystem use
-    """
     parts = []
     if author:
         parts.append(author)
@@ -54,6 +43,11 @@ class QueueStatus(str, Enum):
     CANCELLED = "cancelled"
 
 
+class SearchMode(str, Enum):
+    DIRECT = "direct"
+    UNIVERSAL = "universal"
+
+
 @dataclass
 class QueueItem:
     """Queue item with priority and metadata."""
@@ -70,23 +64,30 @@ class QueueItem:
 
 @dataclass
 class DownloadTask:
-    """Source-agnostic download task for the queue.
-
-    This replaces BookInfo in the queue, providing a unified interface
-    for both Direct Download and Universal modes. The handler uses task_id
-    to fetch whatever source-specific data it needs internally.
-    """
     task_id: str                                # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
     source: str                                 # Handler name ("direct_download", "prowlarr")
     title: str                                  # Display title for queue sidebar
 
     # Display info for queue sidebar
     author: Optional[str] = None
+    year: Optional[str] = None
     format: Optional[str] = None
     size: Optional[str] = None
     preview: Optional[str] = None
     content_type: Optional[str] = None  # "book (fiction)", "audiobook", "magazine", etc.
 
+    # Series info (for library naming templates)
+    series_name: Optional[str] = None
+    series_position: Optional[float] = None  # Float for novellas (e.g., 1.5)
+    subtitle: Optional[str] = None  # Book subtitle for naming templates
+
+    # Hardlinking support
+    original_download_path: Optional[str] = None  # Path in download client (for hardlinking)
+
+    # Search mode - determines post-download processing behavior
+    # See SearchMode enum for behavioral differences
+    search_mode: Optional[SearchMode] = None
+
     # Runtime state
     priority: int = 0
     added_time: float = field(default_factory=time.time)
@@ -105,7 +106,7 @@ class DownloadTask:
         """Build sanitized filename from task metadata."""
         if self.download_path:
             return Path(self.download_path).name
-        return build_filename(self.title, self.author, fmt=self.format)
+        return build_filename(self.title, self.author, self.year, self.format)
 
 
 @dataclass
@@ -130,6 +131,7 @@ class BookInfo:
     status_message: Optional[str] = None  # Detailed status message for UI display
     added_time: Optional[float] = None  # Timestamp when added to queue
     source: str = "direct_download"  # Release source handler to use for downloads
+    source_url: Optional[str] = None  # Link to source page (e.g., Anna's Archive)
 
     def get_filename(self, fallback_url: Optional[str] = None) -> str:
         """Build sanitized filename: 'Author - Title (Year).format'
@@ -144,12 +146,14 @@ class BookInfo:
         """
         # Resolve format if needed
         if not self.format:
-            for url in (self.download_urls[0] if self.download_urls else None, fallback_url):
-                if url:
-                    ext = url.split(".")[-1].lower()
-                    if ext and len(ext) <= 5 and ext.isalnum():
-                        self.format = ext
-                        break
+            urls = [self.download_urls[0]] if self.download_urls else []
+            if fallback_url:
+                urls.append(fallback_url)
+            for url in urls:
+                ext = url.split(".")[-1].lower()
+                if ext and len(ext) <= 5 and ext.isalnum():
+                    self.format = ext
+                    break
 
         return build_filename(self.title, self.author, self.year, self.format)
 

shelfmark/core/naming.py

@@ -0,0 +1,201 @@
+"""Template-based naming for library organization."""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional, Union, Mapping
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+TOKEN_PATTERN = re.compile(
+    r'\{([- ._/\[(]*)'   # prefix: space, dash, dot, underscore, slash, brackets
+    r'([A-Za-z]+)'        # token name
+    r'([- ._/\])]*)\}'    # suffix: space, dash, dot, underscore, slash, brackets
+)
+
+# Characters that are invalid in filenames on various filesystems
+INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def _sanitize(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for filesystem use."""
+    if not name:
+        return ""
+
+    sanitized = INVALID_CHARS.sub('_', name)
+    sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized)  # Strip whitespace and dots
+    sanitized = re.sub(r'_+', '_', sanitized)  # Collapse underscores
+    return sanitized[:max_length]
+
+
+def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for use as a filename or path component."""
+    return _sanitize(name, max_length)
+
+
+# Alias for backwards compatibility
+sanitize_path_component = sanitize_filename
+
+
+def format_series_position(position: Optional[Union[str, int, float]]) -> str:
+    if position is None:
+        return ""
+
+    # Display as integer if whole number
+    if isinstance(position, float) and position.is_integer():
+        return str(int(position))
+
+    return str(position)
+
+
+# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
+PAD_NUMBERS_PATTERN = re.compile(r'\d+')
+
+
+def natural_sort_key(path: Union[str, Path]) -> str:
+    """Generate a sort key with padded numbers for natural sorting."""
+    filename = Path(path).name.lower()
+    return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
+
+
+def assign_part_numbers(
+    files: list[Path],
+    zero_pad_width: int = 2,
+) -> list[tuple[Path, str]]:
+    """Sort files naturally and assign sequential part numbers (1, 2, 3...)."""
+    if not files:
+        return []
+
+    sorted_files = sorted(files, key=natural_sort_key)
+    return [
+        (file_path, str(part_num).zfill(zero_pad_width))
+        for part_num, file_path in enumerate(sorted_files, start=1)
+    ]
+
+
+def parse_naming_template(
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    *,
+    allow_path_separators: bool = True,
+) -> str:
+    if not template:
+        return ""
+
+    # Normalize metadata keys to lowercase for case-insensitive matching
+    normalized = {k.lower(): v for k, v in metadata.items()}
+
+    def replace_token(match: re.Match) -> str:
+        prefix = match.group(1)
+        token_name = match.group(2).lower()
+        suffix = match.group(3)
+
+        # Get the value for this token
+        value = normalized.get(token_name)
+
+        # Special handling for series position
+        if token_name == 'seriesposition':
+            value = format_series_position(value)
+
+        # Convert to string
+        if value is None:
+            value = ""
+        else:
+            value = str(value).strip()
+
+        # If value is empty, return empty string (no prefix/suffix)
+        if not value:
+            return ""
+
+        if not allow_path_separators:
+            value = value.replace("/", "_")
+        # Sanitize the value
+        value = sanitize_filename(value)
+
+        return f"{prefix}{value}{suffix}"
+
+    # Replace all tokens
+    result = TOKEN_PATTERN.sub(replace_token, template)
+
+    # Clean up any double slashes that might result from empty tokens
+    result = re.sub(r'/+', '/', result)
+
+    # Remove leading/trailing slashes
+    result = result.strip('/')
+
+    # Clean up any orphaned separators (e.g., " - " at start/end, or " -  - ")
+    result = re.sub(r'^[\s\-_.]+', '', result)
+    result = re.sub(r'[\s\-_.]+$', '', result)
+    result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
+
+    # Clean up empty parentheses/brackets
+    result = re.sub(r'\(\s*\)', '', result)
+    result = re.sub(r'\[\s*\]', '', result)
+
+    # Final trim of any trailing separators left after cleanup
+    result = re.sub(r'[\s\-_.]+$', '', result)
+
+    return result
+
+
+def build_library_path(
+    base_path: str,
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    extension: Optional[str] = None,
+) -> Path:
+    relative = parse_naming_template(template, metadata, allow_path_separators=True)
+
+    if not relative:
+        # Fallback to title if template produces empty result
+        title = metadata.get('Title') or metadata.get('title') or 'Unknown'
+        relative = sanitize_filename(str(title))
+
+    # Remove any path traversal attempts
+    relative = relative.replace('..', '')
+
+    base = Path(base_path).resolve()
+    full_path = (base / relative).resolve()
+
+    # Verify the path is within the base directory
+    try:
+        full_path.relative_to(base)
+    except ValueError:
+        raise ValueError(f"Path traversal detected: template would escape library directory")
+
+    if extension:
+        ext = extension.lstrip('.')
+        # Don't use with_suffix() - it replaces everything after the first dot
+        # e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
+        full_path = Path(f"{full_path}.{ext}")
+
+    return full_path
+
+
+def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
+    """Check if two paths are on the same filesystem."""
+    path1 = Path(path1)
+    path2 = Path(path2)
+
+    def get_device(p: Path) -> Optional[int]:
+        try:
+            while not p.exists():
+                p = p.parent
+                if p == p.parent:
+                    break
+            return os.stat(p).st_dev
+        except (OSError, PermissionError) as e:
+            logger.debug(f"Cannot stat {p}: {e}")
+            return None
+
+    dev1 = get_device(path1)
+    dev2 = get_device(path2)
+
+    if dev1 is None or dev2 is None:
+        logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
+        return False
+
+    return dev1 == dev2

shelfmark/core/onboarding.py

@@ -0,0 +1,429 @@
+"""
+Onboarding wizard configuration.
+
+Defines the steps and fields for the first-run onboarding experience.
+Reuses field definitions from the settings registry where possible.
+"""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    HeadingField,
+    SettingsField,
+    get_settings_tab,
+    serialize_field,
+    save_config_file,
+    get_setting_value,
+)
+
+logger = setup_logger(__name__)
+
+
+ONBOARDING_STORAGE_KEY = "onboarding_complete"
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def is_onboarding_complete() -> bool:
+    """Check if onboarding has been completed."""
+    config_file = _get_config_dir() / "settings.json"
+    if not config_file.exists():
+        return False
+
+    try:
+        with open(config_file, 'r') as f:
+            config = json.load(f)
+            return config.get(ONBOARDING_STORAGE_KEY, False)
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning(f"Could not read onboarding status from settings.json: {e}")
+        return False
+
+
+def mark_onboarding_complete() -> bool:
+    """Mark onboarding as complete."""
+    try:
+        return save_config_file("general", {ONBOARDING_STORAGE_KEY: True})
+    except Exception as e:
+        logger.error(f"Failed to mark onboarding complete: {e}")
+        return False
+
+
+def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField]:
+    """
+    Extract a specific field from a registered settings tab.
+
+    Args:
+        tab_name: Name of the settings tab (e.g., 'search_mode', 'hardcover')
+        field_key: Key of the field to extract (e.g., 'SEARCH_MODE', 'HARDCOVER_API_KEY')
+
+    Returns:
+        The field if found, None otherwise
+    """
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        logger.warning(f"Settings tab not found: {tab_name}")
+        return None
+
+    for field in tab.fields:
+        if hasattr(field, 'key') and field.key == field_key:
+            return field
+
+    logger.warning(f"Field {field_key} not found in tab {tab_name}")
+    return None
+
+
+def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
+    """
+    Clone a field with optional attribute overrides.
+
+    Useful for customizing labels, descriptions, or defaults for onboarding context.
+    """
+    return replace(field, **overrides)
+
+
+# =============================================================================
+# Step Definitions
+# =============================================================================
+
+
+def get_search_mode_fields() -> List[SettingsField]:
+    """Step 1: Choose search mode - uses actual SEARCH_MODE field from settings."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="welcome_heading",
+            title="Welcome to Shelfmark",
+            description="Let's configure how you want to search for and download books.",
+        ),
+    ]
+
+    # Get the actual SEARCH_MODE field from settings
+    search_mode_field = _get_field_from_tab("search_mode", "SEARCH_MODE")
+    if search_mode_field:
+        # Clone with onboarding-specific description
+        fields.append(_clone_field_with_overrides(
+            search_mode_field,
+            description="Choose how you want to find books.",
+        ))
+
+    return fields
+
+
+def get_metadata_provider_fields() -> List[SettingsField]:
+    """Step 2: Choose metadata provider - uses actual METADATA_PROVIDER field."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="metadata_heading",
+            title="Metadata Provider",
+            description="Choose where to search for book information. You can enable more providers in Settings later.",
+        ),
+    ]
+
+    # Get the actual METADATA_PROVIDER field from settings
+    provider_field = _get_field_from_tab("search_mode", "METADATA_PROVIDER")
+    if provider_field:
+        # Custom options with Hardcover marked as recommended
+        onboarding_options = [
+            {
+                "value": "hardcover",
+                "label": "Hardcover (Recommended)",
+                "description": "Modern book tracking platform with excellent metadata, ratings, and series information. Requires free API key.",
+            },
+            {
+                "value": "openlibrary",
+                "label": "Open Library",
+                "description": "Free, open-source library catalog from the Internet Archive. No API key required.",
+            },
+            {
+                "value": "googlebooks",
+                "label": "Google Books",
+                "description": "Google's book database with good coverage. Requires free API key.",
+            },
+        ]
+
+        # Clone with onboarding-specific options and default
+        fields.append(_clone_field_with_overrides(
+            provider_field,
+            default="hardcover",
+            options=onboarding_options,
+        ))
+
+    return fields
+
+
+def get_hardcover_setup_fields() -> List[SettingsField]:
+    """Step 3a: Configure Hardcover - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="hardcover_setup_heading",
+            title="Hardcover Setup",
+            description="Get your free API key from hardcover.app/account/api",
+            link_url="https://hardcover.app/account/api",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual HARDCOVER_API_KEY field
+    api_key_field = _get_field_from_tab("hardcover", "HARDCOVER_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("hardcover", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_googlebooks_setup_fields() -> List[SettingsField]:
+    """Step 3b: Configure Google Books - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="googlebooks_setup_heading",
+            title="Google Books Setup",
+            description="Get your free API key from Google Cloud Console (APIs & Services > Credentials).",
+            link_url="https://console.cloud.google.com/apis/library/books.googleapis.com",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual GOOGLEBOOKS_API_KEY field
+    api_key_field = _get_field_from_tab("googlebooks", "GOOGLEBOOKS_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("googlebooks", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_prowlarr_fields() -> List[SettingsField]:
+    """Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_heading",
+            title="Prowlarr Integration (Optional)",
+            description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.",
+        ),
+    ]
+
+    # Get actual Prowlarr connection fields
+    prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"]
+    for field_key in prowlarr_fields:
+        field = _get_field_from_tab("prowlarr_config", field_key)
+        if field:
+            fields.append(field)
+
+    return fields
+
+
+def get_prowlarr_indexers_fields() -> List[SettingsField]:
+    """Step 5: Select Prowlarr indexers to search."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_indexers_heading",
+            title="Select Indexers",
+            description="Choose which indexers to search for books. Leave empty to search all available indexers.",
+        ),
+    ]
+
+    # Get the indexers multi-select field
+    indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS")
+    if indexers_field:
+        fields.append(indexers_field)
+
+    return fields
+
+
+# =============================================================================
+# Step Configuration
+# =============================================================================
+
+
+ONBOARDING_STEPS = [
+    {
+        "id": "search_mode",
+        "title": "Search Mode",
+        "tab": "search_mode",
+        "get_fields": get_search_mode_fields,
+    },
+    {
+        "id": "metadata_provider",
+        "title": "Metadata Provider",
+        "tab": "search_mode",
+        "get_fields": get_metadata_provider_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+    },
+    {
+        "id": "hardcover_setup",
+        "title": "Hardcover Setup",
+        "tab": "hardcover",
+        "get_fields": get_hardcover_setup_fields,
+        # Must be universal mode AND hardcover selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "hardcover"},
+        ],
+    },
+    {
+        "id": "googlebooks_setup",
+        "title": "Google Books Setup",
+        "tab": "googlebooks",
+        "get_fields": get_googlebooks_setup_fields,
+        # Must be universal mode AND googlebooks selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "googlebooks"},
+        ],
+    },
+    {
+        "id": "prowlarr",
+        "title": "Prowlarr",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+        "optional": True,
+    },
+    {
+        "id": "prowlarr_indexers",
+        "title": "Indexers",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_indexers_fields,
+        # Only show when Prowlarr is enabled
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "PROWLARR_ENABLED", "value": True},
+        ],
+        "optional": True,
+    },
+]
+
+
+def get_onboarding_config() -> Dict[str, Any]:
+    """
+    Get the full onboarding configuration including steps and current values.
+    """
+    steps = []
+    all_values = {}
+
+    for step_config in ONBOARDING_STEPS:
+        fields = step_config["get_fields"]()
+        tab_name = step_config["tab"]
+
+        # Serialize fields with current values
+        serialized_fields = []
+        for field in fields:
+            serialized = serialize_field(field, tab_name, include_value=True)
+            serialized_fields.append(serialized)
+
+            # Collect values (skip HeadingFields)
+            if hasattr(field, 'key') and field.key and not isinstance(field, HeadingField):
+                value = get_setting_value(field, tab_name)
+                all_values[field.key] = value if value is not None else getattr(field, 'default', '')
+
+        step = {
+            "id": step_config["id"],
+            "title": step_config["title"],
+            "tab": tab_name,
+            "fields": serialized_fields,
+        }
+
+        if "show_when" in step_config:
+            step["showWhen"] = step_config["show_when"]
+        if step_config.get("optional"):
+            step["optional"] = True
+
+        steps.append(step)
+
+    return {
+        "steps": steps,
+        "values": all_values,
+        "complete": is_onboarding_complete(),
+    }
+
+
+def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Save onboarding settings and mark as complete.
+
+    Args:
+        values: Dict of field key -> value
+
+    Returns:
+        Dict with success status and message
+    """
+    try:
+        # Group values by their target tab
+        tab_values: Dict[str, Dict[str, Any]] = {}
+
+        for step_config in ONBOARDING_STEPS:
+            tab_name = step_config["tab"]
+            fields = step_config["get_fields"]()
+
+            for field in fields:
+                if isinstance(field, HeadingField):
+                    continue
+
+                key = field.key
+                if key in values:
+                    if tab_name not in tab_values:
+                        tab_values[tab_name] = {}
+                    tab_values[tab_name][key] = values[key]
+
+        # Save each tab's values
+        for tab_name, tab_data in tab_values.items():
+            if tab_data:
+                save_config_file(tab_name, tab_data)
+                logger.info(f"Saved onboarding settings to {tab_name}: {list(tab_data.keys())}")
+
+        # Enable the selected metadata provider
+        search_mode = values.get("SEARCH_MODE", "direct")
+        if search_mode == "universal":
+            provider = values.get("METADATA_PROVIDER", "hardcover")
+            if provider:
+                # Map provider name to its enabled key
+                enabled_key_map = {
+                    "hardcover": "HARDCOVER_ENABLED",
+                    "openlibrary": "OPENLIBRARY_ENABLED",
+                    "googlebooks": "GOOGLEBOOKS_ENABLED",
+                }
+                enabled_key = enabled_key_map.get(provider, f"{provider.upper()}_ENABLED")
+
+                # Get existing provider config and add enabled flag
+                provider_config = {enabled_key: True}
+
+                # Include API key if provided for that provider
+                if provider == "hardcover" and values.get("HARDCOVER_API_KEY"):
+                    provider_config["HARDCOVER_API_KEY"] = values["HARDCOVER_API_KEY"]
+                elif provider == "googlebooks" and values.get("GOOGLEBOOKS_API_KEY"):
+                    provider_config["GOOGLEBOOKS_API_KEY"] = values["GOOGLEBOOKS_API_KEY"]
+
+                save_config_file(provider, provider_config)
+                logger.info(f"Enabled metadata provider: {provider} with keys: {list(provider_config.keys())}")
+
+        # Mark onboarding as complete
+        mark_onboarding_complete()
+
+        # Refresh config
+        try:
+            from shelfmark.core.config import config
+            config.refresh()
+        except ImportError as e:
+            logger.debug(f"Could not refresh config after onboarding: {e}")
+
+        return {"success": True, "message": "Onboarding complete!"}
+
+    except Exception as e:
+        logger.error(f"Failed to save onboarding settings: {e}")
+        return {"success": False, "message": str(e)}

shelfmark/core/path_mappings.py

@@ -0,0 +1,136 @@
+"""Remote path mapping utilities.
+
+Used when an external download client reports a completed download path that does
+not exist inside the Shelfmark runtime environment (commonly different Docker
+volume mounts).
+
+A mapping rewrites a remote path prefix into a local path prefix.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+
+@dataclass(frozen=True)
+class RemotePathMapping:
+    host: str
+    remote_path: str
+    local_path: str
+
+
+def _normalize_prefix(path: str) -> str:
+    normalized = str(path or "").strip()
+    if not normalized:
+        return ""
+
+    normalized = normalized.replace("\\", "/")
+
+    if normalized != "/":
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def _is_windows_path(path: str) -> bool:
+    """Check if a path looks like a Windows path (has a drive letter like C:/)."""
+    return len(path) >= 2 and path[1] == ":" and path[0].isalpha()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host or "").strip().lower()
+
+
+def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
+    if not value or not isinstance(value, list):
+        return []
+
+    mappings: list[RemotePathMapping] = []
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        host = _normalize_host(row.get("host", ""))
+        remote_path = _normalize_prefix(row.get("remotePath", ""))
+        local_path = _normalize_prefix(row.get("localPath", ""))
+
+        if not host or not remote_path or not local_path:
+            continue
+
+        mappings.append(RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path))
+
+    mappings.sort(key=lambda m: len(m.remote_path), reverse=True)
+    return mappings
+
+
+def remap_remote_to_local_with_match(
+    *,
+    mappings: Iterable[RemotePathMapping],
+    host: str,
+    remote_path: str | Path,
+) -> tuple[Path, bool]:
+    host_normalized = _normalize_host(host)
+    remote_normalized = _normalize_prefix(str(remote_path))
+
+    if not remote_normalized:
+        return Path(str(remote_path)), False
+
+    # Windows paths are case-insensitive, so we need case-insensitive matching
+    # for paths that look like Windows paths (e.g., D:/Torrents)
+    is_windows = _is_windows_path(remote_normalized)
+
+    for mapping in mappings:
+        if _normalize_host(mapping.host) != host_normalized:
+            continue
+
+        remote_prefix = _normalize_prefix(mapping.remote_path)
+        if not remote_prefix:
+            continue
+
+        # For Windows paths, do case-insensitive prefix matching
+        if is_windows:
+            remote_lower = remote_normalized.lower()
+            prefix_lower = remote_prefix.lower()
+            matches = remote_lower == prefix_lower or remote_lower.startswith(prefix_lower + "/")
+        else:
+            matches = remote_normalized == remote_prefix or remote_normalized.startswith(remote_prefix + "/")
+
+        if matches:
+            # Use the length of the original prefix to extract remainder
+            # This preserves the original case in folder names
+            remainder = remote_normalized[len(remote_prefix):]
+            local_prefix = _normalize_prefix(mapping.local_path)
+
+            if remainder.startswith("/"):
+                remainder = remainder[1:]
+
+            remapped = Path(local_prefix) / remainder if remainder else Path(local_prefix)
+            return remapped, True
+
+    return Path(remote_normalized), False
+
+
+def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path) -> Path:
+    remapped, _ = remap_remote_to_local_with_match(
+        mappings=mappings,
+        host=host,
+        remote_path=remote_path,
+    )
+    return remapped
+
+
+def get_client_host_identifier(client: Any) -> Optional[str]:
+    """Return a stable identifier used by the mapping UI.
+
+    Sonarr uses the download client's configured host. Shelfmark currently uses
+    the download client 'name' (e.g. qbittorrent, sabnzbd).
+    """
+
+    name = getattr(client, "name", None)
+    if isinstance(name, str) and name.strip():
+        return name.strip().lower()
+
+    return None

shelfmark/core/prefix_middleware.py

@@ -0,0 +1,31 @@
+"""WSGI middleware for hosting Shelfmark under a URL prefix."""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+
+class PrefixMiddleware:
+    """Strip a configured URL prefix from PATH_INFO before routing."""
+
+    def __init__(self, app, prefix: str, bypass_paths: Optional[Iterable[str]] = None) -> None:
+        self.app = app
+        self.prefix = prefix.rstrip("/")
+        self.bypass_paths = set(bypass_paths or [])
+
+    def __call__(self, environ, start_response):
+        path = environ.get("PATH_INFO", "") or ""
+
+        if path in self.bypass_paths:
+            return self.app(environ, start_response)
+
+        if not self.prefix:
+            return self.app(environ, start_response)
+
+        if path == self.prefix or path.startswith(self.prefix + "/"):
+            environ["SCRIPT_NAME"] = self.prefix
+            environ["PATH_INFO"] = path[len(self.prefix):] or "/"
+            return self.app(environ, start_response)
+
+        start_response("404 Not Found", [("Content-Type", "text/plain")])
+        return [b"Not Found"]

shelfmark/core/queue.py

@@ -7,16 +7,12 @@ from pathlib import Path
 from threading import Lock, Event
 from typing import Dict, List, Optional, Tuple, Any
 
-from cwa_book_downloader.core.config import config as app_config
-from cwa_book_downloader.core.models import QueueStatus, QueueItem, DownloadTask
+from shelfmark.core.config import config as app_config
+from shelfmark.core.models import QueueStatus, QueueItem, DownloadTask
 
 
 class BookQueue:
-    """Thread-safe download queue manager with priority support and cancellation.
-
-    Stores DownloadTask objects which are source-agnostic download descriptors.
-    Works with both Direct Download and Universal modes.
-    """
+    """Thread-safe download queue manager with priority support and cancellation."""
 
     def __init__(self) -> None:
         self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
@@ -33,14 +29,7 @@ class BookQueue:
         return timedelta(seconds=app_config.get("STATUS_TIMEOUT", 3600))
 
     def add(self, task: DownloadTask) -> bool:
-        """Add a download task to the queue.
-
-        Args:
-            task: The download task to queue (includes task_id, priority, etc.)
-
-        Returns:
-            True if added successfully, False if already exists
-        """
+        """Add a download task to the queue. Returns False if already exists."""
         with self._lock:
             task_id = task.task_id
 
@@ -59,11 +48,7 @@ class BookQueue:
             return True
 
     def get_next(self) -> Optional[Tuple[str, Event]]:
-        """Get next task ID from queue with cancellation flag.
-
-        Returns:
-            Tuple of (task_id, cancel_flag) or None if queue is empty
-        """
+        """Get next task ID from queue with cancellation flag."""
         # Use iterative approach to avoid stack overflow if many items are cancelled
         while True:
             try:
@@ -85,14 +70,7 @@ class BookQueue:
                 return None
 
     def get_task(self, task_id: str) -> Optional[DownloadTask]:
-        """Get a task by its ID.
-
-        Args:
-            task_id: The task identifier
-
-        Returns:
-            The DownloadTask if found, None otherwise
-        """
+        """Get a task by its ID."""
         with self._lock:
             return self._task_data.get(task_id)
 
@@ -171,14 +149,7 @@ class BookQueue:
             return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
 
     def cancel_download(self, task_id: str) -> bool:
-        """Cancel a download or clear a completed/errored item.
-
-        Args:
-            task_id: Task identifier to cancel or clear
-
-        Returns:
-            bool: True if cancellation/clearing was successful
-        """
+        """Cancel a download or clear a completed/errored item."""
         with self._lock:
             current_status = self._status.get(task_id)
 
@@ -205,15 +176,7 @@ class BookQueue:
             return False
 
     def set_priority(self, task_id: str, new_priority: int) -> bool:
-        """Change the priority of a queued task.
-
-        Args:
-            task_id: Task identifier
-            new_priority: New priority level (lower = higher priority)
-
-        Returns:
-            bool: True if priority was successfully changed
-        """
+        """Change the priority of a queued task (lower = higher priority)."""
         with self._lock:
             if task_id not in self._status or self._status[task_id] != QueueStatus.QUEUED:
                 return False
@@ -245,14 +208,7 @@ class BookQueue:
             return found
 
     def reorder_queue(self, task_priorities: Dict[str, int]) -> bool:
-        """Bulk reorder queue by setting new priorities.
-
-        Args:
-            task_priorities: Dict mapping task_id to new priority
-
-        Returns:
-            bool: True if reordering was successful
-        """
+        """Bulk reorder queue by mapping task_id to new priority."""
         with self._lock:
             # Extract all items from queue
             all_items = []
@@ -283,39 +239,18 @@ class BookQueue:
             return list(self._active_downloads.keys())
 
     def has_pending_work(self) -> bool:
-        """Check if there are any active downloads or queued items.
-
-        This is useful for determining if the bypasser should stay active
-        even when the UI is closed.
-
-        Returns:
-            bool: True if there are active downloads or queued items
-        """
+        """Check if there are any active downloads or queued items."""
         with self._lock:
-            # Check for active downloads
             if self._active_downloads:
                 return True
-
-            # Check for queued items (excluding cancelled ones)
-            for task_id, status in self._status.items():
-                if status == QueueStatus.QUEUED:
-                    return True
-
-            return False
+            return any(status == QueueStatus.QUEUED for status in self._status.values())
 
     def clear_completed(self) -> int:
-        """Remove all completed, errored, or cancelled tasks from tracking.
-
-        Returns:
-            int: Number of tasks removed
-        """
+        """Remove all completed, errored, or cancelled tasks from tracking."""
+        terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED}
         with self._lock:
-            to_remove = []
-            for task_id, status in self._status.items():
-                if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
-                    to_remove.append(task_id)
+            to_remove = [task_id for task_id, status in self._status.items() if status in terminal_statuses]
 
-            removed_count = len(to_remove)
             for task_id in to_remove:
                 self._status.pop(task_id, None)
                 self._status_timestamps.pop(task_id, None)
@@ -323,14 +258,13 @@ class BookQueue:
                 self._cancel_flags.pop(task_id, None)
                 self._active_downloads.pop(task_id, None)
 
-            return removed_count
+            return len(to_remove)
 
     def refresh(self) -> None:
         """Remove any tasks that are done downloading or have stale status."""
+        terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED}
         with self._lock:
             current_time = datetime.now()
-
-            # Create a list of items to remove to avoid modifying dict during iteration
             to_remove = []
 
             for task_id, status in self._status.items():
@@ -338,28 +272,25 @@ class BookQueue:
                 if not task:
                     continue
 
-                path = task.download_path
-                if path and not Path(path).exists():
+                # Clear stale download paths
+                if task.download_path and not Path(task.download_path).exists():
                     task.download_path = None
-                    path = None
 
-                # Check for completed downloads
-                if status == QueueStatus.AVAILABLE:
-                    if not path:
-                        self._update_status(task_id, QueueStatus.DONE)
+                # Mark available downloads as done if file is gone
+                if status == QueueStatus.AVAILABLE and not task.download_path:
+                    self._update_status(task_id, QueueStatus.DONE)
 
                 # Check for stale status entries
                 last_update = self._status_timestamps.get(task_id)
                 if last_update and (current_time - last_update) > self._status_timeout:
-                    if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]:
+                    if status in terminal_statuses:
                         to_remove.append(task_id)
 
             # Remove stale entries
             for task_id in to_remove:
-                del self._status[task_id]
-                del self._status_timestamps[task_id]
-                if task_id in self._task_data:
-                    del self._task_data[task_id]
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
 
 # Global instance of BookQueue
 book_queue = BookQueue()

shelfmark/core/search_plan.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional
+
+MANUAL_QUERY_MAX_LEN = 256
+
+from shelfmark.core.config import config
+from shelfmark.metadata_providers import (
+    BookMetadata,
+    group_languages_by_localized_title,
+    build_localized_search_titles,
+)
+
+
+@dataclass(frozen=True)
+class ReleaseSearchVariant:
+    """A single search variant (title + author) associated with languages."""
+
+    title: str
+    author: str
+    languages: Optional[List[str]] = None
+
+    @property
+    def query(self) -> str:
+        return " ".join(part for part in [self.title, self.author] if part).strip()
+
+
+@dataclass(frozen=True)
+class ReleaseSearchPlan:
+    """Pre-computed search inputs shared across release sources."""
+
+    languages: Optional[List[str]]
+    isbn_candidates: List[str]
+    author: str
+    title_variants: List[ReleaseSearchVariant]
+    grouped_title_variants: List[ReleaseSearchVariant]
+    manual_query: Optional[str] = None
+
+    @property
+    def primary_query(self) -> str:
+        return self.title_variants[0].query if self.title_variants else ""
+
+
+def _normalize_languages(languages: Optional[List[str]]) -> Optional[List[str]]:
+    if not languages:
+        default = config.BOOK_LANGUAGE
+        if not default:
+            return None
+        return [str(lang).strip() for lang in default if str(lang).strip()]
+
+    normalized: List[str] = []
+    for lang in languages:
+        if not lang:
+            continue
+        s = str(lang).strip()
+        if not s:
+            continue
+        normalized.append(s)
+
+    if any(lang.lower() == "all" for lang in normalized):
+        return None
+
+    return normalized or None
+
+
+def _pick_search_author(book: BookMetadata) -> str:
+    if book.search_author:
+        return book.search_author
+
+    if not book.authors:
+        return ""
+
+    first = book.authors[0]
+    if "," in first:
+        first = first.split(",")[0].strip()
+
+    return first
+
+
+def _pick_search_title(book: BookMetadata) -> str:
+    return book.search_title or book.title
+
+
+def build_release_search_plan(
+    book: BookMetadata,
+    languages: Optional[List[str]] = None,
+    manual_query: Optional[str] = None,
+) -> ReleaseSearchPlan:
+    resolved_languages = _normalize_languages(languages)
+
+    resolved_manual_query = None
+    if manual_query:
+        resolved_manual_query = manual_query.strip()[:MANUAL_QUERY_MAX_LEN] or None
+
+    author = _pick_search_author(book)
+    base_title = _pick_search_title(book)
+
+    if resolved_manual_query:
+        # Manual override: use the raw query as-is (no language/title expansion).
+        variant = ReleaseSearchVariant(title=resolved_manual_query, author="", languages=None)
+        return ReleaseSearchPlan(
+            languages=resolved_languages,
+            isbn_candidates=[],
+            author="",
+            title_variants=[variant],
+            grouped_title_variants=[variant],
+            manual_query=resolved_manual_query,
+        )
+
+    isbn_candidates: List[str] = []
+    if book.isbn_13:
+        isbn_candidates.append(book.isbn_13)
+    if book.isbn_10 and book.isbn_10 not in isbn_candidates:
+        isbn_candidates.append(book.isbn_10)
+
+    titles_by_language = book.titles_by_language or None
+    if book.search_title and titles_by_language:
+        titles_by_language = {
+            k: v
+            for k, v in titles_by_language.items()
+            if str(k).strip().lower() not in {"en", "eng", "english"}
+        }
+
+    grouped = group_languages_by_localized_title(
+        base_title=base_title,
+        languages=resolved_languages,
+        titles_by_language=titles_by_language,
+    )
+
+    grouped_variants: List[ReleaseSearchVariant] = [
+        ReleaseSearchVariant(title=title, author=author, languages=langs)
+        for title, langs in grouped
+        if title
+    ]
+
+    expanded_titles = build_localized_search_titles(
+        base_title=base_title,
+        languages=resolved_languages,
+        titles_by_language=titles_by_language,
+        excluded_languages={"en", "eng", "english"},
+    )
+
+    title_variants: List[ReleaseSearchVariant] = [
+        ReleaseSearchVariant(title=title, author=author, languages=None)
+        for title in expanded_titles
+        if title
+    ]
+
+    # If no titles could be built, fall back to ISBN queries.
+    if not title_variants and isbn_candidates:
+        title_variants = [
+            ReleaseSearchVariant(title=isbn, author="", languages=None)
+            for isbn in isbn_candidates
+        ]
+
+    return ReleaseSearchPlan(
+        languages=resolved_languages,
+        isbn_candidates=isbn_candidates,
+        author=author,
+        title_variants=title_variants,
+        grouped_title_variants=grouped_variants,
+        manual_query=None,
+    )

shelfmark/core/settings_registry.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Type, Union
 from threading import Lock
 
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -24,9 +24,10 @@ class FieldBase:
     env_supported: bool = True            # Whether this setting can be set via ENV var (False = UI-only)
     disabled: bool = False                # Whether field is disabled/greyed out
     disabled_reason: str = ""             # Explanation shown when disabled
-    show_when: Optional[Dict[str, Any]] = None  # Conditional visibility: {"field": "key", "value": "expected"}
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
     disabled_when: Optional[Dict[str, Any]] = None  # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
     requires_restart: bool = False        # Whether changing this setting requires a container restart
+    universal_only: bool = False          # Only show in Universal search mode (hide in Direct mode)
 
     def get_env_var_name(self) -> str:
         """Get the environment variable name for this field."""
@@ -70,6 +71,7 @@ class SelectField(FieldBase):
     """Single-choice dropdown."""
     # Options can be a list or a callable that returns a list (for lazy evaluation)
     options: Any = field(default_factory=list)  # [{value: "", label: ""}] or callable
+    filter_by_field: Optional[str] = None  # Field key whose value filters options via childOf property
 
 
 @dataclass
@@ -78,46 +80,44 @@ class MultiSelectField(FieldBase):
     # Options can be a list or a callable that returns a list (for lazy evaluation)
     options: Any = field(default_factory=list)  # [{value: "", label: ""}] or callable
     default: List[str] = field(default_factory=list)
+    variant: str = "pills"  # "pills" (default) or "dropdown" for checkbox dropdown style
 
 
 @dataclass
 class OrderableListField(FieldBase):
-    """
-    Drag-and-drop reorderable list with enable/disable toggles.
-
-    A generic field for any ordered list of items where each item can be
-    enabled or disabled. Used for source priority, format preference, etc.
-
-    Options define the available items:
-        [{"id": "item1", "label": "Item 1", "description": "...",
-          "disabledReason": "...", "isLocked": False}, ...]
-
-    Value is stored as:
-        [{"id": "item1", "enabled": True}, {"id": "item2", "enabled": False}, ...]
-    """
     # Options can be a list or a callable that returns a list (for lazy evaluation)
-    # Each option: {id, label, description?, disabledReason?, isLocked?}
+    # Each option: {id, label, description?, disabledReason?, isLocked?, section?, isPinned?}
+    # - isLocked: toggle is disabled (can't enable/disable)
+    # - isPinned: can't be reordered (but toggle may still work if not also isLocked)
     options: Any = field(default_factory=list)
     # Default value: [{id, enabled}, ...] in priority order
     default: List[Dict[str, Any]] = field(default_factory=list)
 
 
 @dataclass
-class ActionButton:
-    """
-    Button that triggers a callback function.
+class TableField(FieldBase):
+    """Editable table of structured rows."""
 
-    Used for actions like "Test Connection" that execute code
-    and return success/error status.
-    """
+    # Column definitions: [{key, label, type, placeholder?, options?, defaultValue?}, ...]
+    columns: Any = field(default_factory=list)  # list or callable
+
+    # Value format: list of objects
+    default: List[Dict[str, Any]] = field(default_factory=list)
+
+    add_label: str = "Add"
+    empty_message: str = ""
+
+
+@dataclass
+class ActionButton:
     key: str                              # Action identifier
     label: str                            # Button text
     description: str = ""                 # Help text
     style: str = "default"                # "default", "primary", "danger"
-    callback: Optional[Callable[[], Dict[str, Any]]] = None  # Returns {"success": bool, "message": str}
+    callback: Optional[Callable[..., Dict[str, Any]]] = None  # Returns {"success": bool, "message": str}
     disabled: bool = False                # Whether button is disabled/greyed out
     disabled_reason: str = ""             # Explanation shown when disabled
-    show_when: Optional[Dict[str, Any]] = None  # Conditional visibility: {"field": "key", "value": "expected"}
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
     disabled_when: Optional[Dict[str, Any]] = None  # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
 
     def get_field_type(self) -> str:
@@ -137,7 +137,8 @@ class HeadingField:
     description: str = ""                 # Description text (supports markdown-style links)
     link_url: str = ""                    # Optional URL for a link
     link_text: str = ""                   # Text for the link (defaults to URL if not provided)
-    show_when: Optional[Dict[str, Any]] = None  # Conditional visibility: {"field": "key", "value": "expected"}
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    universal_only: bool = False          # Only show in Universal search mode (hide in Direct mode)
 
     def get_field_type(self) -> str:
         return "HeadingField"
@@ -179,20 +180,6 @@ def register_group(
     icon: Optional[str] = None,
     order: int = 100
 ) -> None:
-    """
-    Register a settings group.
-
-    Groups are collapsible containers for related settings tabs.
-
-    Args:
-        name: Internal name for the group (e.g., "direct_download")
-        display_name: Display name in UI (e.g., "Direct Download")
-        icon: Optional icon name for the UI
-        order: Sort order (lower numbers appear first)
-
-    Example:
-        register_group("direct_download", "Direct Download", icon="download", order=20)
-    """
     with _REGISTRY_LOCK:
         group = SettingsGroup(
             name=name,
@@ -211,25 +198,6 @@ def register_settings(
     order: int = 100,
     group: Optional[str] = None
 ):
-    """
-    Decorator to register settings for a plugin/module.
-
-    The decorated function should return a list of SettingsField objects.
-
-    Args:
-        name: Internal name for the settings tab (e.g., "hardcover")
-        display_name: Display name in UI (e.g., "Hardcover")
-        icon: Optional icon name for the UI
-        order: Sort order (lower numbers appear first)
-        group: Optional group name this tab belongs to
-
-    Example:
-        @register_settings("hardcover", "Hardcover", icon="book", order=20, group="metadata_providers")
-        def hardcover_settings():
-            return [
-                PasswordField(key="HARDCOVER_API_KEY", label="API Key", required=True),
-            ]
-    """
     def decorator(func: Callable[[], List[SettingsField]]):
         with _REGISTRY_LOCK:
             fields = func()
@@ -252,28 +220,6 @@ def register_on_save(
     tab_name: str,
     handler: Callable[[Dict[str, Any]], Dict[str, Any]]
 ) -> None:
-    """
-    Register a custom on_save handler for a settings tab.
-
-    The handler is called before saving settings and can:
-    - Validate values (return {"error": True, "message": "..."})
-    - Transform values (e.g., hash passwords)
-    - Add computed values
-
-    Args:
-        tab_name: The settings tab name to register the handler for.
-        handler: Callable that takes values dict and returns:
-                 {"error": bool, "message": str (if error), "values": dict}
-
-    Example:
-        def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
-            password = values.pop("password", "")
-            if password:
-                values["password_hash"] = hash_password(password)
-            return {"error": False, "values": values}
-
-        register_on_save("security", _on_save_security)
-    """
     with _REGISTRY_LOCK:
         _ON_SAVE_HANDLERS[tab_name] = handler
         logger.debug(f"Registered on_save handler for tab: {tab_name}")
@@ -301,18 +247,17 @@ def list_registered_settings() -> List[str]:
 
 def _get_config_dir() -> Path:
     """Get the config directory path."""
-    from cwa_book_downloader.config.env import CONFIG_DIR
+    from shelfmark.config.env import CONFIG_DIR
     return Path(CONFIG_DIR)
 
 
 def _get_config_file_path(tab_name: str) -> Path:
     """Get the config file path for a settings tab."""
     config_dir = _get_config_dir()
-    if tab_name == "general":
+    # Core settings tabs share the main settings.json file
+    if tab_name in ("general", "search_mode"):
         return config_dir / "settings.json"
-    else:
-        plugins_dir = config_dir / "plugins"
-        return plugins_dir / f"{tab_name}.json"
+    return config_dir / "plugins" / f"{tab_name}.json"
 
 
 def _ensure_config_dir(tab_name: str) -> None:
@@ -322,15 +267,6 @@ def _ensure_config_dir(tab_name: str) -> None:
 
 
 def load_config_file(tab_name: str) -> Dict[str, Any]:
-    """
-    Load settings from a config file.
-
-    Args:
-        tab_name: The settings tab name.
-
-    Returns:
-        Dict of setting key -> value from config file.
-    """
     config_path = _get_config_file_path(tab_name)
 
     if not config_path.exists():
@@ -345,16 +281,6 @@ def load_config_file(tab_name: str) -> Dict[str, Any]:
 
 
 def save_config_file(tab_name: str, values: Dict[str, Any]) -> bool:
-    """
-    Save settings to a config file.
-
-    Args:
-        tab_name: The settings tab name.
-        values: Dict of setting key -> value to save.
-
-    Returns:
-        True if save succeeded, False otherwise.
-    """
     try:
         _ensure_config_dir(tab_name)
         config_path = _get_config_file_path(tab_name)
@@ -373,15 +299,78 @@ def save_config_file(tab_name: str, values: Dict[str, Any]) -> bool:
         return False
 
 
+def initialize_default_configs() -> bool:
+    """Initialize config files with default values on first startup.
+
+    Creates config files for all settings tabs that don't have one yet,
+    populating them with field default values. This ensures config files
+    exist from first startup rather than only being created on explicit save.
+
+    Returns:
+        True if initialization succeeded or was skipped (already initialized),
+        False if there was an error accessing the config directory.
+    """
+    try:
+        config_dir = _get_config_dir()
+
+        # Check if config directory exists and is writable
+        if not config_dir.exists():
+            logger.warning(f"Config directory does not exist: {config_dir}")
+            return False
+
+        # Test writability
+        test_file = config_dir / ".write_test"
+        try:
+            test_file.touch()
+            test_file.unlink()
+        except (OSError, PermissionError) as e:
+            logger.warning(f"Config directory is not writable: {config_dir} - {e}")
+            return False
+
+        initialized_tabs = []
+
+        for tab in get_all_settings_tabs():
+            config_path = _get_config_file_path(tab.name)
+
+            # Skip if config file already exists
+            if config_path.exists():
+                continue
+
+            # Collect default values for all fields
+            defaults = {}
+            for field in tab.fields:
+                # Skip non-value fields
+                if isinstance(field, (ActionButton, HeadingField)):
+                    continue
+
+                # Only include fields that have a non-None default
+                if field.default is not None:
+                    defaults[field.key] = field.default
+
+            # Create config file with defaults if we have any
+            if defaults:
+                _ensure_config_dir(tab.name)
+                try:
+                    with open(config_path, 'w') as f:
+                        json.dump(defaults, f, indent=2)
+                    initialized_tabs.append(tab.name)
+                except Exception as e:
+                    logger.error(f"Failed to initialize config for {tab.name}: {e}")
+
+        if initialized_tabs:
+            logger.info(f"Initialized default configs for: {initialized_tabs}")
+
+        return True
+
+    except Exception as e:
+        logger.error(f"Error during config initialization: {e}")
+        return False
+
+
 def sync_env_to_config() -> None:
-    """
-    Sync environment variable values to config files.
+    # Initialize default configs first (for fresh installs)
+    initialize_default_configs()
 
-    This ensures that when ENV vars are set, their values are persisted to config.
-    When ENV vars are later removed, the config file retains the last known values.
-
-    Called once during application startup.
-    """
     for tab in get_all_settings_tabs():
         values_to_sync = {}
 
@@ -408,20 +397,119 @@ def sync_env_to_config() -> None:
             save_config_file(tab.name, values_to_sync)
             logger.debug(f"Synced {len(values_to_sync)} ENV values to {tab.name} config: {list(values_to_sync.keys())}")
 
+    migrate_legacy_settings()
+
+
+def migrate_legacy_settings() -> None:
+    """Migrate legacy settings to new unified file destination format.
+
+    Maps old settings to new:
+    - PROCESSING_MODE + USE_BOOK_TITLE -> FILE_ORGANIZATION
+    - INGEST_DIR / LIBRARY_PATH -> DESTINATION
+    - LIBRARY_TEMPLATE -> TEMPLATE
+    - USE_CONTENT_TYPE_DIRECTORIES -> AA_CONTENT_TYPE_ROUTING
+    - INGEST_DIR_* -> AA_CONTENT_TYPE_DIR_*
+    - TORRENT_HARDLINK -> HARDLINK_TORRENTS / HARDLINK_TORRENTS_AUDIOBOOK
+    """
+    # Load existing downloads config
+    downloads_config = load_config_file("downloads")
+    source_config = load_config_file("download_sources")
+
+    # Skip migration if already using new settings
+    if "FILE_ORGANIZATION" in downloads_config or "DESTINATION" in downloads_config:
+        return
+
+    # Skip migration if no legacy settings exist (fresh install)
+    legacy_keys = {
+        "PROCESSING_MODE", "INGEST_DIR", "LIBRARY_PATH", "USE_BOOK_TITLE",
+        "LIBRARY_TEMPLATE", "PROCESSING_MODE_AUDIOBOOK", "INGEST_DIR_AUDIOBOOK",
+        "LIBRARY_PATH_AUDIOBOOK", "LIBRARY_TEMPLATE_AUDIOBOOK", "TORRENT_HARDLINK",
+        "USE_CONTENT_TYPE_DIRECTORIES",
+    }
+    if not any(key in downloads_config for key in legacy_keys):
+        return
+
+    migrated_downloads = {}
+    migrated_sources = {}
+
+    # === BOOKS MIGRATION ===
+    old_mode = downloads_config.get("PROCESSING_MODE", "ingest")
+    old_ingest_dir = downloads_config.get("INGEST_DIR", "/cwa-book-ingest")
+    old_library_path = downloads_config.get("LIBRARY_PATH", "")
+    old_use_book_title = downloads_config.get("USE_BOOK_TITLE", True)
+    old_library_template = downloads_config.get("LIBRARY_TEMPLATE", "{Author}/{Title}")
+
+    # Map PROCESSING_MODE + USE_BOOK_TITLE -> FILE_ORGANIZATION
+    if old_mode == "library":
+        migrated_downloads["FILE_ORGANIZATION"] = "organize"
+        migrated_downloads["DESTINATION"] = old_library_path or "/books"
+        migrated_downloads["TEMPLATE"] = old_library_template
+    else:
+        if old_use_book_title:
+            migrated_downloads["FILE_ORGANIZATION"] = "rename"
+            migrated_downloads["TEMPLATE"] = "{Author} - {Title} ({Year})"
+        else:
+            migrated_downloads["FILE_ORGANIZATION"] = "none"
+        migrated_downloads["DESTINATION"] = old_ingest_dir
+
+    # === AUDIOBOOKS MIGRATION ===
+    old_mode_ab = downloads_config.get("PROCESSING_MODE_AUDIOBOOK", "ingest")
+    old_ingest_dir_ab = downloads_config.get("INGEST_DIR_AUDIOBOOK", "")
+    old_library_path_ab = downloads_config.get("LIBRARY_PATH_AUDIOBOOK", "")
+    old_library_template_ab = downloads_config.get("LIBRARY_TEMPLATE_AUDIOBOOK", "{Author}/{Title}")
+
+    if old_mode_ab == "library":
+        migrated_downloads["FILE_ORGANIZATION_AUDIOBOOK"] = "organize"
+        migrated_downloads["DESTINATION_AUDIOBOOK"] = old_library_path_ab or ""
+        migrated_downloads["TEMPLATE_AUDIOBOOK"] = old_library_template_ab
+    else:
+        migrated_downloads["FILE_ORGANIZATION_AUDIOBOOK"] = "rename"
+        migrated_downloads["TEMPLATE_AUDIOBOOK"] = "{Author} - {Title}"
+        if old_ingest_dir_ab:
+            migrated_downloads["DESTINATION_AUDIOBOOK"] = old_ingest_dir_ab
+
+    # === HARDLINK MIGRATION ===
+    old_torrent_hardlink = downloads_config.get("TORRENT_HARDLINK")
+    if old_torrent_hardlink is not None:
+        # Books default to False (ingest folder use case)
+        # Audiobooks default to True (library folder use case)
+        # But if explicitly set, apply to both
+        migrated_downloads["HARDLINK_TORRENTS"] = old_torrent_hardlink
+        migrated_downloads["HARDLINK_TORRENTS_AUDIOBOOK"] = old_torrent_hardlink
+
+    # === CONTENT-TYPE ROUTING MIGRATION ===
+    old_use_content_type = downloads_config.get("USE_CONTENT_TYPE_DIRECTORIES", False)
+    if old_use_content_type:
+        migrated_sources["AA_CONTENT_TYPE_ROUTING"] = True
+
+        # Map old keys to new keys
+        content_type_mapping = {
+            "INGEST_DIR_BOOK_FICTION": "AA_CONTENT_TYPE_DIR_FICTION",
+            "INGEST_DIR_BOOK_NON_FICTION": "AA_CONTENT_TYPE_DIR_NON_FICTION",
+            "INGEST_DIR_BOOK_UNKNOWN": "AA_CONTENT_TYPE_DIR_UNKNOWN",
+            "INGEST_DIR_MAGAZINE": "AA_CONTENT_TYPE_DIR_MAGAZINE",
+            "INGEST_DIR_COMIC_BOOK": "AA_CONTENT_TYPE_DIR_COMIC",
+            "INGEST_DIR_STANDARDS_DOCUMENT": "AA_CONTENT_TYPE_DIR_STANDARDS",
+            "INGEST_DIR_MUSICAL_SCORE": "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+            "INGEST_DIR_OTHER": "AA_CONTENT_TYPE_DIR_OTHER",
+        }
+
+        for old_key, new_key in content_type_mapping.items():
+            old_value = downloads_config.get(old_key, "")
+            if old_value:
+                migrated_sources[new_key] = old_value
+
+    # Save migrated settings
+    if migrated_downloads:
+        save_config_file("downloads", migrated_downloads)
+        logger.info(f"Migrated download settings: {list(migrated_downloads.keys())}")
+
+    if migrated_sources:
+        save_config_file("download_sources", migrated_sources)
+        logger.info(f"Migrated content-type routing settings: {list(migrated_sources.keys())}")
+
 
 def get_setting_value(field: SettingsField, tab_name: str) -> Any:
-    """
-    Get the current value for a settings field.
-
-    Priority: env var > config file > default
-
-    Args:
-        field: The settings field.
-        tab_name: The settings tab name (for config file lookup).
-
-    Returns:
-        The resolved value.
-    """
     if isinstance(field, (ActionButton, HeadingField)):
         return None  # Actions and headings don't have values
 
@@ -461,6 +549,14 @@ def _parse_env_value(value: str, field: SettingsField) -> Any:
         except json.JSONDecodeError:
             logger.warning(f"Invalid JSON for {field.key}, using default")
             return field.default
+    elif isinstance(field, TableField):
+        # Parse JSON array: [{"col": "value"}, ...]
+        try:
+            parsed = json.loads(value)
+            return parsed if isinstance(parsed, list) else field.default
+        except json.JSONDecodeError:
+            logger.warning(f"Invalid JSON for {field.key}, using default")
+            return field.default
     else:
         return value
 
@@ -470,12 +566,9 @@ def is_value_from_env(field: SettingsField) -> bool:
     if isinstance(field, (ActionButton, HeadingField)):
         return False
     # UI-only settings never come from ENV (env_supported=False)
-    # Default to True for backwards compatibility
-    env_supported = getattr(field, 'env_supported', True)
-    if env_supported is False:
+    if not getattr(field, 'env_supported', True):
         return False
-    env_var_name = field.get_env_var_name()
-    return env_var_name in os.environ
+    return field.get_env_var_name() in os.environ
 
 
 def serialize_field(field: SettingsField, tab_name: str, include_value: bool = True) -> Dict[str, Any]:
@@ -492,7 +585,7 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
     """
     # HeadingField has a different structure - handle separately
     if isinstance(field, HeadingField):
-        result = {
+        result: Dict[str, Any] = {
             "key": field.key,
             "type": field.get_field_type(),
             "title": field.title,
@@ -503,9 +596,11 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
             result["linkText"] = field.link_text or field.link_url
         if field.show_when:
             result["showWhen"] = field.show_when
+        if field.universal_only:
+            result["universalOnly"] = True
         return result
 
-    result = {
+    result: Dict[str, Any] = {
         "key": field.key,
         "label": field.label,
         "type": field.get_field_type(),
@@ -516,15 +611,13 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
         "requiresRestart": getattr(field, 'requires_restart', False),
     }
 
-    # Add conditional visibility if specified
-    show_when = getattr(field, 'show_when', None)
-    if show_when:
-        result["showWhen"] = show_when
-
-    # Add conditional disable if specified
-    disabled_when = getattr(field, 'disabled_when', None)
-    if disabled_when:
-        result["disabledWhen"] = disabled_when
+    # Add optional properties if set
+    if getattr(field, 'show_when', None):
+        result["showWhen"] = field.show_when
+    if getattr(field, 'disabled_when', None):
+        result["disabledWhen"] = field.disabled_when
+    if getattr(field, 'universal_only', False):
+        result["universalOnly"] = True
 
     # Add type-specific properties
     if isinstance(field, TextField):
@@ -537,20 +630,56 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
         result["min"] = field.min_value
         result["max"] = field.max_value
         result["step"] = field.step
-    elif isinstance(field, (SelectField, MultiSelectField)):
+    elif isinstance(field, SelectField):
         # Support callable options for lazy evaluation (avoids circular imports)
         options = field.options() if callable(field.options) else field.options
         result["options"] = options
+        if field.default is not None:
+            result["default"] = field.default
+        if field.filter_by_field:
+            result["filterByField"] = field.filter_by_field
+    elif isinstance(field, MultiSelectField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+        result["variant"] = field.variant
     elif isinstance(field, OrderableListField):
         # Support callable options for lazy evaluation (avoids circular imports)
         options = field.options() if callable(field.options) else field.options
         result["options"] = options
+    elif isinstance(field, TableField):
+        columns = field.columns() if callable(field.columns) else field.columns
+        result["columns"] = columns
+        result["addLabel"] = field.add_label
+        result["emptyMessage"] = field.empty_message
     elif isinstance(field, ActionButton):
         result["style"] = field.style
         result["description"] = field.description
 
     if include_value and not isinstance(field, (ActionButton, HeadingField)):
         value = get_setting_value(field, tab_name)
+
+        # Ensure select values are serialized as strings so the frontend can
+        # reliably match against string option values.
+        if isinstance(field, SelectField) and value is not None:
+            value = str(value)
+        elif isinstance(field, MultiSelectField):
+            if value is None:
+                value = []
+            elif isinstance(value, list):
+                value = [str(v) for v in value]
+            elif isinstance(value, str):
+                # Support legacy/manual configs where MultiSelect values were saved
+                # as comma-separated strings.
+                value = [v.strip() for v in value.split(",") if v.strip()]
+            else:
+                value = []
+        elif isinstance(field, TableField):
+            if value is None:
+                value = []
+            elif not isinstance(value, list):
+                value = []
+
         result["value"] = value if value is not None else ""
         result["fromEnv"] = is_value_from_env(field)
 
@@ -619,7 +748,7 @@ def execute_action(tab_name: str, action_key: str, current_values: Optional[Dict
                 try:
                     # Check if callback accepts current_values parameter
                     sig = inspect.signature(field.callback)
-                    if 'current_values' in sig.parameters:
+                    if "current_values" in sig.parameters:
                         return field.callback(current_values=current_values or {})
                     else:
                         return field.callback()
@@ -640,7 +769,7 @@ def _sync_metadata_provider_selection() -> None:
     the first enabled provider if the current selection is invalid.
     """
     try:
-        from cwa_book_downloader.metadata_providers import sync_metadata_provider_selection
+        from shelfmark.metadata_providers import sync_metadata_provider_selection
         sync_metadata_provider_selection()
     except ImportError:
         pass  # Metadata providers module not available
@@ -654,7 +783,7 @@ def _apply_dns_settings(config) -> None:
     a container restart.
     """
     try:
-        from cwa_book_downloader.download import network
+        from shelfmark.download import network
 
         provider = config.get("CUSTOM_DNS", "auto")
         use_doh = config.get("USE_DOH", False)
@@ -674,19 +803,6 @@ def _apply_dns_settings(config) -> None:
 
 
 def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Update settings for a tab.
-
-    Only updates values that are not set via environment variables.
-
-    Args:
-        tab_name: The settings tab name.
-        values: Dict of key -> value to update.
-
-    Returns:
-        Dict with "success" (bool), "message" (str), "updated" (list of keys),
-        and "requiresRestart" (bool) indicating if any changed setting requires restart.
-    """
     tab = get_settings_tab(tab_name)
     if not tab:
         return {"success": False, "message": f"Unknown settings tab: {tab_name}", "updated": [], "requiresRestart": False}
@@ -752,16 +868,22 @@ def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]:
     # Save to config file
     if save_config_file(tab_name, values_to_save):
         # Refresh the config singleton so live settings take effect immediately
+        config_obj = None
         try:
-            from cwa_book_downloader.core.config import config
-            config.refresh()
+            from shelfmark.core.config import config as config_obj
+
+            config_obj.refresh()
         except ImportError:
-            pass  # Config module not yet available during initial setup
+            config_obj = None  # Config module not yet available during initial setup
 
         # Apply DNS settings changes live (network tab)
         dns_keys = {"CUSTOM_DNS", "CUSTOM_DNS_MANUAL", "USE_DOH"}
-        if tab_name == "network" and dns_keys.intersection(values_to_save.keys()):
-            _apply_dns_settings(config)
+        if (
+            config_obj is not None
+            and tab_name == "network"
+            and dns_keys.intersection(values_to_save.keys())
+        ):
+            _apply_dns_settings(config_obj)
 
         # Sync metadata provider selection when a provider's enabled state changes
         tab = get_settings_tab(tab_name)

shelfmark/core/utils.py

@@ -0,0 +1,196 @@
+"""Shared utility functions for the Shelfmark."""
+
+import base64
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+
+def normalize_http_url(
+    url: Optional[str],
+    *,
+    default_scheme: str = "http",
+    strip_trailing_slash: bool = True,
+    allow_special: tuple[str, ...] = (),
+) -> str:
+    """Normalize a configured HTTP URL for requests and links."""
+    if not isinstance(url, str):
+        return ""
+
+    normalized = url.strip()
+    if not normalized:
+        return ""
+
+    if (normalized.startswith("\"") and normalized.endswith("\"")) or (
+        normalized.startswith("'") and normalized.endswith("'")
+    ):
+        normalized = normalized[1:-1].strip()
+        if not normalized:
+            return ""
+
+    if allow_special:
+        special_map = {
+            value.lower(): value
+            for value in allow_special
+            if isinstance(value, str)
+        }
+        special_match = special_map.get(normalized.lower())
+        if special_match is not None:
+            return special_match
+
+    if normalized.startswith(("/", "./", "../")):
+        return normalized
+
+    if "://" not in normalized:
+        scheme = default_scheme.strip().rstrip(":/")
+        if scheme:
+            normalized = f"{scheme}://{normalized}"
+
+    if strip_trailing_slash:
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def normalize_base_path(value: Optional[str]) -> str:
+    """Normalize a URL base path for reverse proxy subpath deployments."""
+    if not isinstance(value, str):
+        return ""
+
+    path = value.strip()
+    if not path:
+        return ""
+
+    if "://" in path:
+        parsed = urlparse(path)
+        path = parsed.path or ""
+
+    if not path or path == "/":
+        return ""
+
+    if not path.startswith("/"):
+        path = "/" + path
+
+    return path.rstrip("/")
+
+
+def is_audiobook(content_type: Optional[str]) -> bool:
+    """Check if content type indicates an audiobook."""
+    return bool(content_type and "audiobook" in content_type.lower())
+
+
+CONTENT_TYPES = [
+    "book (fiction)",
+    "book (non-fiction)",
+    "book (unknown)",
+    "magazine",
+    "comic book",
+    "audiobook",
+    "standards document",
+    "musical score",
+    "other",
+]
+
+# Maps AA content types to their config keys for content-type routing
+# Used when AA_CONTENT_TYPE_ROUTING is enabled
+_AA_CONTENT_TYPE_TO_CONFIG_KEY = {
+    "book (fiction)": "AA_CONTENT_TYPE_DIR_FICTION",
+    "book (non-fiction)": "AA_CONTENT_TYPE_DIR_NON_FICTION",
+    "book (unknown)": "AA_CONTENT_TYPE_DIR_UNKNOWN",
+    "magazine": "AA_CONTENT_TYPE_DIR_MAGAZINE",
+    "comic book": "AA_CONTENT_TYPE_DIR_COMIC",
+    "audiobook": "AA_CONTENT_TYPE_DIR_AUDIOBOOK",
+    "standards document": "AA_CONTENT_TYPE_DIR_STANDARDS",
+    "musical score": "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+    "other": "AA_CONTENT_TYPE_DIR_OTHER",
+}
+
+# Legacy mapping - kept for backwards compatibility during migration
+_LEGACY_CONTENT_TYPE_TO_CONFIG_KEY = {
+    "book (fiction)": "INGEST_DIR_BOOK_FICTION",
+    "book (non-fiction)": "INGEST_DIR_BOOK_NON_FICTION",
+    "book (unknown)": "INGEST_DIR_BOOK_UNKNOWN",
+    "magazine": "INGEST_DIR_MAGAZINE",
+    "comic book": "INGEST_DIR_COMIC_BOOK",
+    "audiobook": "INGEST_DIR_AUDIOBOOK",
+    "standards document": "INGEST_DIR_STANDARDS_DOCUMENT",
+    "musical score": "INGEST_DIR_MUSICAL_SCORE",
+    "other": "INGEST_DIR_OTHER",
+}
+
+
+def get_destination(is_audiobook: bool = False) -> Path:
+    """Get base destination directory. Audiobooks fall back to main destination."""
+    from shelfmark.core.config import config
+
+    if is_audiobook:
+        # Audiobook destination with fallback to main destination
+        audiobook_dest = config.get("DESTINATION_AUDIOBOOK", "")
+        if audiobook_dest:
+            return Path(audiobook_dest)
+
+    # Main destination (also fallback for audiobooks)
+    # Check new setting first, then legacy INGEST_DIR
+    destination = config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books")
+    return Path(destination)
+
+
+def get_aa_content_type_dir(content_type: Optional[str] = None) -> Optional[Path]:
+    """Get override directory for AA content-type routing if configured."""
+    from shelfmark.core.config import config
+
+    # Check if content-type routing is enabled (new or legacy setting)
+    if not config.get("AA_CONTENT_TYPE_ROUTING", False) and not config.get("USE_CONTENT_TYPE_DIRECTORIES", False):
+        return None
+
+    if not content_type:
+        return None
+
+    content_type_lower = content_type.lower().strip()
+
+    # Try new AA-specific config keys first, then legacy keys
+    for mapping in (_AA_CONTENT_TYPE_TO_CONFIG_KEY, _LEGACY_CONTENT_TYPE_TO_CONFIG_KEY):
+        config_key = mapping.get(content_type_lower)
+        if config_key:
+            custom_dir = config.get(config_key, "")
+            if custom_dir:
+                return Path(custom_dir)
+
+    return None
+
+
+def get_ingest_dir(content_type: Optional[str] = None) -> Path:
+    """DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead."""
+    from shelfmark.core.config import config
+
+    # Check new DESTINATION setting first, then legacy INGEST_DIR
+    default_ingest_dir = Path(config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books"))
+
+    if not content_type:
+        return default_ingest_dir
+
+    # Check for content-type override
+    override_dir = get_aa_content_type_dir(content_type)
+    if override_dir:
+        return override_dir
+
+    return default_ingest_dir
+
+
+def transform_cover_url(cover_url: Optional[str], cache_id: str) -> Optional[str]:
+    """Transform external cover URL to local proxy URL when caching is enabled."""
+    if not cover_url:
+        return cover_url
+
+    # Skip if already a local URL (starts with /)
+    if cover_url.startswith('/'):
+        return cover_url
+
+    # Check if cover caching is enabled
+    from shelfmark.config.env import is_covers_cache_enabled
+    if not is_covers_cache_enabled():
+        return cover_url
+
+    # Encode the original URL and create a proxy URL
+    encoded_url = base64.urlsafe_b64encode(cover_url.encode()).decode()
+    return f"/api/covers/{cache_id}?url={encoded_url}"

shelfmark/download/archive.py

@@ -0,0 +1,239 @@
+"""Archive extraction utilities for downloaded book archives."""
+
+import os
+import shutil
+import zipfile
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.postprocess.policy import (
+    get_supported_audiobook_formats,
+    get_supported_formats,
+)
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.fs import atomic_write
+
+logger = setup_logger(__name__)
+
+
+# Check for rarfile availability at module load
+try:
+    import rarfile
+
+    RAR_AVAILABLE = True
+except ImportError:
+    RAR_AVAILABLE = False
+    logger.warning("rarfile not installed - RAR extraction disabled")
+
+
+class ArchiveExtractionError(Exception):
+    """Raised when archive extraction fails."""
+
+    pass
+
+
+class PasswordProtectedError(ArchiveExtractionError):
+    """Raised when archive requires a password."""
+
+    pass
+
+
+class CorruptedArchiveError(ArchiveExtractionError):
+    """Raised when archive is corrupted."""
+
+    pass
+
+
+def is_archive(file_path: Path) -> bool:
+    """Check if file is a supported archive format."""
+    suffix = file_path.suffix.lower().lstrip(".")
+    return suffix in ("zip", "rar")
+
+
+def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> bool:
+    """Check if file matches user's supported formats setting based on content type."""
+    ext = file_path.suffix.lower().lstrip(".")
+    if check_audiobook(content_type):
+        supported_formats = get_supported_audiobook_formats()
+    else:
+        supported_formats = get_supported_formats()
+    return ext in supported_formats
+
+
+# All known ebook extensions (superset of what user might enable)
+ALL_EBOOK_EXTENSIONS = {'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr', '.doc', '.docx', '.rtf', '.txt'}
+
+# All known audio extensions (superset of what user might enable for audiobooks)
+ALL_AUDIO_EXTENSIONS = {'.m4b', '.mp3', '.m4a', '.aac', '.flac', '.ogg', '.wma', '.wav', '.opus'}
+
+
+def _filter_files(
+    extracted_files: List[Path],
+    content_type: Optional[str] = None,
+) -> Tuple[List[Path], List[Path], List[Path]]:
+    """Filter files by content type. Returns (matched, rejected_format, other)."""
+    is_audiobook = check_audiobook(content_type)
+    known_extensions = ALL_AUDIO_EXTENSIONS if is_audiobook else ALL_EBOOK_EXTENSIONS
+
+    matched_files = []
+    rejected_format_files = []
+    other_files = []
+
+    for file_path in extracted_files:
+        if _is_supported_file(file_path, content_type):
+            matched_files.append(file_path)
+        elif file_path.suffix.lower() in known_extensions:
+            rejected_format_files.append(file_path)
+        else:
+            other_files.append(file_path)
+
+    return matched_files, rejected_format_files, other_files
+
+
+def extract_archive(
+    archive_path: Path,
+    output_dir: Path,
+    content_type: Optional[str] = None,
+) -> Tuple[List[Path], List[str], List[Path]]:
+    """Extract archive and filter by content type. Returns (matched, warnings, rejected)."""
+    suffix = archive_path.suffix.lower().lstrip(".")
+
+    if suffix == "zip":
+        extracted_files, warnings = _extract_zip(archive_path, output_dir)
+    elif suffix == "rar":
+        extracted_files, warnings = _extract_rar(archive_path, output_dir)
+    else:
+        raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
+
+    is_audiobook = check_audiobook(content_type)
+    file_type_label = "audiobook" if is_audiobook else "book"
+
+    # Filter files based on content type
+    matched_files, rejected_files, other_files = _filter_files(extracted_files, content_type)
+
+    # Delete rejected files (valid formats but not enabled by user)
+    for rejected_file in rejected_files:
+        try:
+            rejected_file.unlink()
+            logger.debug(f"Deleted rejected {file_type_label} file: {rejected_file.name}")
+        except OSError as e:
+            logger.warning(f"Failed to delete rejected {file_type_label} file {rejected_file}: {e}")
+
+    if rejected_files:
+        rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+        warnings.append(f"Skipped {len(rejected_files)} {file_type_label}(s) with unsupported format: {', '.join(rejected_exts)}")
+
+    # Delete other files (images, html, etc)
+    for other_file in other_files:
+        try:
+            other_file.unlink()
+            logger.debug(f"Deleted non-{file_type_label} file: {other_file.name}")
+        except OSError as e:
+            logger.warning(f"Failed to delete non-{file_type_label} file {other_file}: {e}")
+
+    if other_files:
+        warnings.append(f"Skipped {len(other_files)} non-{file_type_label} file(s)")
+
+    return matched_files, warnings, rejected_files
+
+
+def extract_archive_raw(
+    archive_path: Path,
+    output_dir: Path,
+) -> Tuple[List[Path], List[str]]:
+    """Extract archive without filtering (returns all extracted files)."""
+    suffix = archive_path.suffix.lower().lstrip(".")
+
+    if suffix == "zip":
+        return _extract_zip(archive_path, output_dir)
+    if suffix == "rar":
+        return _extract_rar(archive_path, output_dir)
+
+    raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
+
+
+def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
+    """Extract files from ZipFile or RarFile to output_dir with security checks."""
+    extracted_files = []
+
+    for info in archive.infolist():
+        if info.is_dir():
+            continue
+
+        # Use only filename, strip directory path (security: prevent path traversal)
+        filename = Path(info.filename).name
+        if not filename:
+            continue
+
+        # Security: reject filenames with null bytes or path separators
+        # Check both / and \ since archives may be created on different OSes
+        if "\x00" in filename or "/" in filename or "\\" in filename:
+            logger.warning(f"Skipping suspicious filename in archive: {info.filename!r}")
+            continue
+
+        # Extract to output_dir with flat structure
+        target_path = output_dir / filename
+
+        # Security: verify resolved path stays within output directory (defense-in-depth)
+        try:
+            target_path.resolve().relative_to(output_dir.resolve())
+        except ValueError:
+            logger.warning(f"Path traversal attempt blocked: {info.filename!r}")
+            continue
+
+        with archive.open(info) as src:
+            data = src.read()
+        final_path = atomic_write(target_path, data)
+        extracted_files.append(final_path)
+        logger.debug(f"Extracted: {filename}")
+
+    return extracted_files
+
+
+def _extract_zip(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
+    """Extract files from a ZIP archive."""
+    try:
+        with zipfile.ZipFile(archive_path, "r") as zf:
+            # Check for password protection
+            for info in zf.infolist():
+                if info.flag_bits & 0x1:  # Encrypted flag
+                    raise PasswordProtectedError("ZIP archive is password protected")
+
+            # Test archive integrity
+            bad_file = zf.testzip()
+            if bad_file:
+                raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
+
+            return _extract_files_from_archive(zf, output_dir), []
+
+    except zipfile.BadZipFile as e:
+        raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
+    except PermissionError as e:
+        raise ArchiveExtractionError(f"Permission denied: {e}")
+
+
+def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
+    """Extract files from a RAR archive."""
+    if not RAR_AVAILABLE:
+        raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
+
+    try:
+        with rarfile.RarFile(archive_path, "r") as rf:
+            # Check for password protection
+            if rf.needs_password():
+                raise PasswordProtectedError("RAR archive is password protected")
+
+            # Test archive integrity
+            rf.testrar()
+
+            return _extract_files_from_archive(rf, output_dir), []
+
+    except rarfile.BadRarFile as e:
+        raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
+    except rarfile.RarCannotExec:
+        raise ArchiveExtractionError("unrar binary not found - install unrar package")
+    except PermissionError as e:
+        raise ArchiveExtractionError(f"Permission denied: {e}")
+
+

shelfmark/download/fs.py

@@ -0,0 +1,409 @@
+"""Atomic filesystem operations for concurrent-safe file handling.
+
+These utilities handle file collisions atomically, avoiding TOCTOU race conditions
+when multiple workers may try to write to the same path simultaneously.
+"""
+
+import errno
+import os
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.permissions_debug import log_transfer_permission_context
+
+logger = setup_logger(__name__)
+
+
+
+_VERIFY_IO_WAIT_SECONDS = 3.0
+
+
+def _verify_transfer_size(
+    dest: Path,
+    expected_size: int,
+    action: str,
+) -> None:
+    """Verify file transfer completed successfully.
+
+    Some filesystems (especially remote NAS/CIFS/NFS) can report stale sizes briefly
+    after large writes. Do a second stat after a short delay before declaring failure.
+    """
+    actual_size = dest.stat().st_size
+    if actual_size == expected_size:
+        return
+
+    logger.debug(
+        f"File {action} size mismatch, waiting for filesystem sync: {dest} "
+        f"({actual_size} != {expected_size})"
+    )
+    time.sleep(_VERIFY_IO_WAIT_SECONDS)
+
+    actual_size = dest.stat().st_size
+    if actual_size != expected_size:
+        raise IOError(
+            f"File {action} incomplete, data loss may have occurred. "
+            f"'{dest}' was {actual_size} bytes instead of expected {expected_size}."
+        )
+
+
+def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
+    """Write data to a file with atomic collision detection.
+
+    If the destination already exists, retries with counter suffix (_1, _2, etc.)
+    until a unique path is found.
+
+    Args:
+        dest_path: Desired destination path
+        data: Bytes to write
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually written (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            # O_CREAT | O_EXCL fails atomically if file exists
+            fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+            try:
+                os.write(fd, data)
+            finally:
+                os.close(fd)
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            continue
+
+    raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}")
+
+
+def _is_permission_error(e: Exception) -> bool:
+    """Check if exception is a permission error (including NFS/SMB issues)."""
+    return isinstance(e, PermissionError) or (isinstance(e, OSError) and e.errno == errno.EPERM)
+
+
+def _system_op(op: str, source: Path, dest: Path) -> None:
+    """Execute system command (mv or cp) as final fallback."""
+    logger.warning("Attempting system %s as final fallback: %s -> %s", op, source, dest)
+    subprocess.run(
+        [op, "-f", str(source), str(dest)],
+        check=True,
+        capture_output=True,
+        text=True
+    )
+
+
+def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
+    """Handle NFS/SMB permission errors by falling back to copyfile -> system op."""
+    expected_size = source.stat().st_size
+
+    try:
+        # Fallback 1: copy content only
+        shutil.copyfile(str(source), str(dest))
+        _verify_transfer_size(dest, expected_size, "copy")
+
+        if is_move:
+            source.unlink()
+        return
+
+    except Exception as copy_error:
+        # Clean up failed copy attempt if it exists
+        dest.unlink(missing_ok=True)
+
+        if _is_permission_error(copy_error):
+            log_transfer_permission_context("nfs_fallback_copyfile", source=source, dest=dest, error=copy_error)
+        logger.error("Fallback copyfile failed (%s -> %s): %s", source, dest, copy_error)
+
+        # Fallback 2: system command
+        op = "mv" if is_move else "cp"
+        try:
+            _system_op(op, source, dest)
+            # Best-effort verify after external command.
+            if dest.exists():
+                _verify_transfer_size(dest, expected_size, op)
+            if is_move:
+                source.unlink(missing_ok=True)
+        except subprocess.CalledProcessError as sys_error:
+            log_transfer_permission_context("nfs_fallback_system", source=source, dest=dest, error=sys_error)
+            logger.error("System %s failed (%s -> %s): %s", op, source, dest, sys_error.stderr)
+            dest.unlink(missing_ok=True)
+            raise
+
+
+def _claim_destination(path: Path) -> bool:
+    """Atomically claim a destination path by creating a placeholder file.
+
+    Returns True if the placeholder was created. Caller must replace or unlink it.
+    """
+    try:
+        fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+    except FileExistsError:
+        return False
+    else:
+        os.close(fd)
+        return True
+
+
+def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Move a file with collision detection.
+
+    Uses os.rename() for same-filesystem moves (atomic, triggers inotify events),
+    falls back to exclusive create + shutil.move for cross-filesystem moves.
+
+    Note: We use os.rename() instead of hardlink+unlink because os.rename()
+    triggers proper inotify IN_MOVED_TO events that file watchers (like Calibre's
+    auto-add) rely on to detect new files.
+
+    Args:
+        source_path: Source file to move
+        dest_path: Desired destination path
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually moved (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+
+        # Check for existing file (os.rename would overwrite on Unix)
+        claimed = False
+        if try_path.exists():
+            # Some filesystems can report false positives for exists() with
+            # special characters. Probe with O_EXCL to confirm.
+            claimed = _claim_destination(try_path)
+            if not claimed:
+                continue
+
+        try:
+            # os.rename is atomic on same filesystem and triggers inotify events
+            if claimed:
+                os.replace(str(source_path), str(try_path))
+            else:
+                os.rename(str(source_path), str(try_path))
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            # Race condition: file created between exists() check and rename()
+            if claimed:
+                try_path.unlink(missing_ok=True)
+            continue
+        except OSError as e:
+            # Cross-filesystem - fall back to exclusive create + verified copy + delete.
+            if e.errno != errno.EXDEV:
+                if claimed:
+                    try_path.unlink(missing_ok=True)
+                raise
+
+            expected_size = source_path.stat().st_size
+
+            try:
+                if not claimed:
+                    # Claim destination path atomically.
+                    fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+                    os.close(fd)
+
+                # Copy to a temp file first, then replace to avoid partial files.
+                temp_path = try_path.parent / f".{try_path.name}.tmp"
+                try:
+                    try:
+                        shutil.copy2(str(source_path), str(temp_path))
+                    except (PermissionError, OSError) as copy_error:
+                        if _is_permission_error(copy_error):
+                            logger.debug(
+                                "Permission error during move-copy, falling back to copyfile (%s -> %s): %s",
+                                source_path,
+                                temp_path,
+                                copy_error,
+                            )
+                            _perform_nfs_fallback(source_path, temp_path, is_move=False)
+                        else:
+                            raise
+
+                    temp_path.replace(try_path)
+                    _verify_transfer_size(try_path, expected_size, "move")
+                    source_path.unlink()
+
+                    if attempt > 0:
+                        logger.info(f"File collision resolved: {try_path.name}")
+                    return try_path
+
+                except Exception:
+                    try_path.unlink(missing_ok=True)
+                    temp_path.unlink(missing_ok=True)
+                    raise
+
+            except FileExistsError:
+                continue
+            except (PermissionError, OSError) as e:
+                if _is_permission_error(e):
+                    log_transfer_permission_context(
+                        "atomic_move",
+                        source=source_path,
+                        dest=try_path,
+                        error=e,
+                    )
+                    logger.debug(
+                        "Permission error during move, falling back to copyfile (%s -> %s): %s",
+                        source_path,
+                        try_path,
+                        e,
+                    )
+                    try:
+                        _perform_nfs_fallback(source_path, try_path, is_move=True)
+                        if attempt > 0:
+                            logger.info(f"File collision resolved (fallback): {try_path.name}")
+                        return try_path
+                    except Exception as fallback_error:
+                        logger.error(
+                            "NFS fallback also failed (%s -> %s): %s",
+                            source_path,
+                            try_path,
+                            fallback_error,
+                        )
+                        raise e from fallback_error
+                raise
+
+    raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
+
+
+def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Create a hardlink with atomic collision detection.
+
+    Args:
+        source_path: Source file to link from
+        dest_path: Desired destination path for the link
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where link was actually created (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            os.link(str(source_path), str(try_path))
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            continue
+        except OSError as e:
+            if _is_permission_error(e) or e.errno in (errno.EXDEV, errno.EMLINK):
+                if _is_permission_error(e):
+                    log_transfer_permission_context(
+                        "atomic_hardlink",
+                        source=source_path,
+                        dest=try_path,
+                        error=e,
+                    )
+                logger.debug(
+                    "Hardlink failed (%s), falling back to copy: %s -> %s",
+                    e,
+                    source_path,
+                    dest_path,
+                )
+                return atomic_copy(source_path, dest_path, max_attempts=max_attempts)
+            raise
+
+    raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
+
+
+def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Copy a file with atomic collision detection.
+
+    Uses exclusive create to claim destination, then copies via temp file
+    to avoid partial files on failure.
+
+    Args:
+        source_path: Source file to copy
+        dest_path: Desired destination path
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually copied (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            # Atomically claim the destination by creating an exclusive file
+            fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+            os.close(fd)
+            
+            # Copy to temp file first, then replace to avoid partial files
+            temp_path = try_path.parent / f".{try_path.name}.tmp"
+            try:
+                try:
+                    shutil.copy2(str(source_path), str(temp_path))
+                except (PermissionError, OSError) as e:
+                    # Handle NFS permission errors immediately here
+                    if _is_permission_error(e):
+                        log_transfer_permission_context(
+                            "atomic_copy",
+                            source=source_path,
+                            dest=temp_path,
+                            error=e,
+                        )
+                        logger.debug(
+                            "Permission error during copy, falling back to copyfile (%s -> %s): %s",
+                            source_path,
+                            temp_path,
+                            e,
+                        )
+                        try:
+                            _perform_nfs_fallback(source_path, temp_path, is_move=False)
+                        except Exception as fallback_error:
+                            logger.error(
+                                "NFS fallback also failed (%s -> %s): %s",
+                                source_path,
+                                temp_path,
+                                fallback_error,
+                            )
+                            raise e from fallback_error
+                    else:
+                        raise
+                
+                temp_path.replace(try_path)
+                _verify_transfer_size(try_path, source_path.stat().st_size, "copy")
+                if attempt > 0:
+                    logger.info(f"File collision resolved: {try_path.name}")
+                return try_path
+            except Exception:
+                try_path.unlink(missing_ok=True)
+                temp_path.unlink(missing_ok=True)
+                raise
+        except FileExistsError:
+            continue
+
+    raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")

shelfmark/download/http.py

@@ -10,53 +10,105 @@ from urllib.parse import urlparse
 import requests
 from tqdm import tqdm
 
-from cwa_book_downloader.download import network
-from cwa_book_downloader.config.env import USE_CF_BYPASS, USING_EXTERNAL_BYPASSER
-from cwa_book_downloader.core.config import config as app_config
-from cwa_book_downloader.core.logger import setup_logger
-
-# Import bypasser if enabled
-if USE_CF_BYPASS:
-    if USING_EXTERNAL_BYPASSER:
-        from cwa_book_downloader.bypass.external_bypasser import get_bypassed_page
-        # External bypasser doesn't share cookies/UA
-        get_cf_cookies_for_domain = lambda domain: {}
-        get_cf_user_agent_for_domain = lambda domain: None
-    else:
-        from cwa_book_downloader.bypass.internal_bypasser import get_bypassed_page, get_cf_cookies_for_domain, get_cf_user_agent_for_domain
+from shelfmark.download import network
+from shelfmark.download.network import get_proxies
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
+# Bypasser modules are imported lazily to support dynamic selection based on config
+_internal_bypasser = None
+_external_bypasser = None
+
+
+def _get_internal_bypasser():
+    """Lazy import of internal bypasser module."""
+    global _internal_bypasser
+    if _internal_bypasser is None:
+        try:
+            from shelfmark.bypass import internal_bypasser
+            _internal_bypasser = internal_bypasser
+        except ImportError as e:
+            raise RuntimeError(
+                f"Failed to import internal bypasser: {e}. "
+                "Check that all dependencies are installed. "
+                "You may need to disable CF bypass or use the external bypasser."
+            ) from e
+    return _internal_bypasser
+
+
+def _get_external_bypasser():
+    """Lazy import of external bypasser module."""
+    global _external_bypasser
+    if _external_bypasser is None:
+        try:
+            from shelfmark.bypass import external_bypasser
+            _external_bypasser = external_bypasser
+        except ImportError as e:
+            raise RuntimeError(
+                f"Failed to import external bypasser: {e}. "
+                "Check that the external bypasser is properly configured."
+            ) from e
+    return _external_bypasser
+
+
+def _is_using_external_bypasser() -> bool:
+    """Check if external bypasser is configured (reads from config, not just env)."""
+    return app_config.get("USING_EXTERNAL_BYPASSER", False)
+
+
+def _is_cf_bypass_enabled() -> bool:
+    """Check if Cloudflare bypass is enabled."""
+    return app_config.get("USE_CF_BYPASS", True)
+
+
+def get_bypassed_page(url, selector=None, cancel_flag=None):
+    """Wrapper that delegates to the appropriate bypasser based on config."""
+    if _is_using_external_bypasser():
+        return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag)
+    return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
+
+
+def get_cf_cookies_for_domain(domain):
+    """Get CF cookies - only available with internal bypasser."""
+    if _is_using_external_bypasser():
+        logger.debug(f"External bypasser in use, CF cookies not available for {domain}")
+        return {}
+    return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
+
+
+def get_cf_user_agent_for_domain(domain):
+    """Get CF user agent - only available with internal bypasser."""
+    if _is_using_external_bypasser():
+        logger.debug(f"External bypasser in use, CF user agent not available for {domain}")
+        return None
+    return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
+
+
+def _apply_cf_bypass(url: str, headers: dict) -> dict:
+    """Apply CF bypass cookies and user agent if available.
+
+    Modifies headers in-place with the stored user agent (if available).
+    Returns cookies dict to use with the request.
+    """
+    if not _is_cf_bypass_enabled():
+        return {}
+
+    parsed = urlparse(url)
+    hostname = parsed.hostname or ""
+    cookies = get_cf_cookies_for_domain(hostname)
+    stored_ua = get_cf_user_agent_for_domain(hostname)
+    if stored_ua:
+        headers['User-Agent'] = stored_ua
+    return cookies
+
+
 # Network settings
 REQUEST_TIMEOUT = (5, 10)  # (connect, read)
 MAX_DOWNLOAD_RETRIES = 2
 MAX_RESUME_ATTEMPTS = 3
 
-
-def _get_proxies() -> dict:
-    """Get current proxy configuration from config singleton."""
-    proxy_mode = app_config.get("PROXY_MODE", "none")
-
-    if proxy_mode == "socks5":
-        socks_proxy = app_config.get("SOCKS5_PROXY", "")
-        if socks_proxy:
-            return {"http": socks_proxy, "https": socks_proxy}
-    elif proxy_mode == "http":
-        proxies = {}
-        http_proxy = app_config.get("HTTP_PROXY", "")
-        https_proxy = app_config.get("HTTPS_PROXY", "")
-        if http_proxy:
-            proxies["http"] = http_proxy
-        if https_proxy:
-            proxies["https"] = https_proxy
-        elif http_proxy:
-            # Fallback: use HTTP proxy for HTTPS if HTTPS proxy not specified
-            proxies["https"] = http_proxy
-        return proxies
-
-    return {}
-
-
 RETRYABLE_CODES = (429, 500, 502, 503, 504)
 CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout,
                      requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError)
@@ -64,6 +116,9 @@ DOWNLOAD_HEADERS = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
     'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
 }
 
 
@@ -97,7 +152,7 @@ def _is_retryable_error(e: Exception) -> bool:
     if isinstance(e, CONNECTION_ERRORS):
         return True
     status = _get_status_code(e)
-    return status in RETRYABLE_CODES if status else False
+    return status is not None and status in RETRYABLE_CODES
 
 
 def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]:
@@ -120,8 +175,15 @@ def html_get_page(
     use_bypasser: bool = False,
     selector: Optional[network.AAMirrorSelector] = None,
     cancel_flag: Optional[Event] = None,
+    status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
+    allow_bypasser_fallback: bool = True,
 ) -> str:
-    """Fetch HTML content from a URL with retry mechanism."""
+    """Fetch HTML content from a URL with retry mechanism.
+
+    Args:
+        allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
+            instead of switching to the bypasser. Use for search operations.
+    """
     retry = retry if retry is not None else app_config.MAX_RETRY
     selector = selector or network.AAMirrorSelector()
     original_url = url
@@ -135,8 +197,10 @@ def html_get_page(
             return ""
 
         try:
-            if use_bypasser_now and USE_CF_BYPASS:
-                logger.info(f"GET (bypasser): {current_url}")
+            if use_bypasser_now and _is_cf_bypass_enabled():
+                logger.debug(f"GET (bypasser): {current_url}")
+                if status_callback:
+                    status_callback("resolving", "Bypassing protection")
                 try:
                     result = get_bypassed_page(current_url, selector, cancel_flag)
                     return result or ""
@@ -144,18 +208,11 @@ def html_get_page(
                     logger.warning(f"Bypasser error: {type(e).__name__}: {e}")
                     return ""
 
-            logger.info(f"GET: {current_url}")
+            logger.debug(f"GET: {current_url}")
             # Try with CF cookies/UA if available (from previous bypass)
-            cookies = {}
             headers = {}
-            if USE_CF_BYPASS:
-                parsed = urlparse(current_url)
-                hostname = parsed.hostname or ""
-                cookies = get_cf_cookies_for_domain(hostname)
-                stored_ua = get_cf_user_agent_for_domain(hostname)
-                if stored_ua:
-                    headers['User-Agent'] = stored_ua
-            response = requests.get(current_url, proxies=_get_proxies(), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
+            cookies = _apply_cf_bypass(current_url, headers)
+            response = requests.get(current_url, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
             response.raise_for_status()
             time.sleep(1)
             return response.text
@@ -165,7 +222,16 @@ def html_get_page(
 
             # 403 = Cloudflare/DDoS-Guard protection
             if status == 403:
-                if USE_CF_BYPASS and not use_bypasser_now:
+                # If bypasser fallback is disabled, try mirrors instead
+                if not allow_bypasser_fallback:
+                    new_url = _try_rotation(original_url, current_url, selector)
+                    if new_url:
+                        current_url = new_url
+                        continue
+                    logger.warning(f"403 error, mirrors exhausted: {current_url}")
+                    return ""
+
+                if _is_cf_bypass_enabled() and not use_bypasser_now:
                     # Before switching to bypasser, check if cookies have become available
                     # (another concurrent download may have completed bypass and extracted cookies)
                     parsed = urlparse(current_url)
@@ -175,6 +241,8 @@ def html_get_page(
                         logger.debug(f"403 but cookies now available - retrying with cookies: {current_url}")
                         continue
                     logger.info(f"403 detected; switching to bypasser: {current_url}")
+                    if status_callback:
+                        status_callback("resolving", "Bypassing protection...")
                     use_bypasser_now = True
                     continue
                 logger.warning(f"403 error, giving up: {current_url}")
@@ -237,21 +305,8 @@ def download_url(
 
             logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
             # Try with CF cookies/UA if available
-            cookies = {}
-            if USE_CF_BYPASS:
-                parsed = urlparse(current_url)
-                hostname = parsed.hostname or ""
-                cookies = get_cf_cookies_for_domain(hostname)
-                # Use stored UA - Cloudflare ties cf_clearance to the UA that solved the challenge
-                stored_ua = get_cf_user_agent_for_domain(hostname)
-                if stored_ua:
-                    headers['User-Agent'] = stored_ua
-                    logger.debug(f"Using stored UA for {hostname}")
-                else:
-                    logger.debug(f"No stored UA available for {hostname}")
-                if cookies:
-                    logger.debug(f"Using {len(cookies)} cookies for {hostname}: {list(cookies.keys())}")
-            response = requests.get(current_url, stream=True, proxies=_get_proxies(), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
+            cookies = _apply_cf_bypass(current_url, headers)
+            response = requests.get(current_url, stream=True, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
             response.raise_for_status()
 
             if status_callback:
@@ -286,7 +341,7 @@ def download_url(
             retryable = _is_retryable_error(e)
 
             # Z-Library 403 - try refreshing cookies via bypasser once before giving up
-            if status == 403 and USE_CF_BYPASS and not zlib_cookie_refresh_attempted:
+            if status == 403 and _is_cf_bypass_enabled() and not zlib_cookie_refresh_attempted:
                 parsed = urlparse(current_url)
                 if parsed.hostname and 'z-lib' in parsed.hostname and referer:
                     zlib_cookie_refresh_attempted = True
@@ -309,14 +364,14 @@ def download_url(
             if status == 429:
                 logger.info(f"Rate limited (429) - trying next source")
                 if status_callback:
-                    status_callback("resolving", "Server busy, trying next...")
+                    status_callback("resolving", "Server busy, trying next")
                 return None
 
             # Timeout - don't retry, server likely overloaded
             if isinstance(e, requests.exceptions.Timeout):
                 logger.warning(f"Timeout: {current_url} - skipping to next source")
                 if status_callback:
-                    status_callback("resolving", "Server timed out, trying next...")
+                    status_callback("resolving", "Server timed out, trying next")
                 return None
 
             # Try to resume if we got some data
@@ -358,17 +413,10 @@ def _try_resume(
 
         try:
             # Try with CF cookies/UA if available
-            cookies = {}
             resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'}
-            if USE_CF_BYPASS:
-                parsed = urlparse(url)
-                hostname = parsed.hostname or ""
-                cookies = get_cf_cookies_for_domain(hostname)
-                stored_ua = get_cf_user_agent_for_domain(hostname)
-                if stored_ua:
-                    resume_headers['User-Agent'] = stored_ua
+            cookies = _apply_cf_bypass(url, resume_headers)
             response = requests.get(
-                url, stream=True, proxies=_get_proxies(), timeout=REQUEST_TIMEOUT,
+                url, stream=True, proxies=get_proxies(url), timeout=REQUEST_TIMEOUT,
                 headers=resume_headers, cookies=cookies
             )
             

shelfmark/download/network.py

@@ -1,6 +1,6 @@
 """DNS rotation, mirror selection, and network utilities."""
 
-import os
+import fnmatch
 import requests
 import urllib.request
 from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable
@@ -8,18 +8,67 @@ import socket
 import dns.resolver
 from socket import AddressFamily, SocketKind
 import urllib.parse
-import ssl
 import ipaddress
 
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.config.settings import AA_BASE_URL, AA_AVAILABLE_URLS
-from cwa_book_downloader.config import settings as config
-from cwa_book_downloader.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.config import config as app_config
+from shelfmark.core.utils import normalize_http_url
 from datetime import datetime, timedelta
 
 
-def _get_proxies() -> dict:
-    """Get current proxy configuration from config singleton."""
+def _get_no_proxy_patterns() -> List[str]:
+    """Get list of NO_PROXY patterns from config."""
+    no_proxy = app_config.get("NO_PROXY", "")
+    if not no_proxy:
+        return []
+    return [p.strip().lower() for p in no_proxy.split(",") if p.strip()]
+
+
+def should_bypass_proxy(url: str) -> bool:
+    """Check if a URL should bypass the proxy based on NO_PROXY patterns.
+
+    Supports:
+    - Exact hostname match: localhost, myhost.local
+    - Wildcard prefix: *.local matches foo.local
+    - Wildcard suffix: 10.* matches 10.1.2.3
+    """
+    if not url:
+        return False
+
+    patterns = _get_no_proxy_patterns()
+    if not patterns:
+        return False
+
+    # Extract hostname from URL
+    try:
+        parsed = urllib.parse.urlparse(url)
+        hostname = (parsed.hostname or "").lower()
+    except Exception as e:
+        logger.debug(f"Failed to parse URL for proxy bypass check: {url} - {e}")
+        return False
+
+    if not hostname:
+        return False
+
+    for pattern in patterns:
+        # Use fnmatch for wildcard matching (supports * and ?)
+        if fnmatch.fnmatch(hostname, pattern):
+            return True
+
+    return False
+
+
+def get_proxies(url: str = "") -> dict:
+    """Get current proxy configuration from config singleton.
+
+    Args:
+        url: Optional URL to check against NO_PROXY patterns.
+             If provided and matches a pattern, returns empty dict.
+    """
+    # Check NO_PROXY bypass first
+    if url and should_bypass_proxy(url):
+        return {}
+
     proxy_mode = app_config.get("PROXY_MODE", "none")
 
     if proxy_mode == "socks5":
@@ -107,13 +156,6 @@ def _notify_dns_rotation(provider_name: str, servers: List[str], doh_url: str) -
         except Exception as e:
             logger.warning(f"DNS rotation callback {callback.__name__} failed: {e}")
 
-def _agent_debug_log(code: str, source: str, reason: str, meta: Optional[dict] = None) -> None:
-    """Lightweight debug hook for automated runs; safe no-op on failure."""
-    try:
-        logger.debug(f"[agent] code={code} source={source} reason={reason} meta={meta or {}}")
-    except Exception as exc:
-        # Avoid raising inside debug logger
-        logger.debug(f"[agent] log failed: {exc}")
 
 def _load_state():
     """Return current in-memory network state (no disk persistence)."""
@@ -133,7 +175,8 @@ def _save_state(aa_url=None, dns_provider=None):
 
 # AA URL failover state
 _current_aa_url_index = 0
-_aa_urls = AA_AVAILABLE_URLS.copy()
+_aa_urls: List[str] = []  # Initialized lazily in _initialize_aa_state()
+_aa_base_url: str = ""  # Current active AA URL
 
 def _ensure_initialized() -> None:
     """Lazy guard so runtime setup happens once and late calls still work."""
@@ -234,34 +277,41 @@ def _decode_port(port: Union[str, bytes, int, None]) -> int:
     """Convert port to integer, handling various input types."""
     if port is None:
         return 0
-    if isinstance(port, (str, bytes)):
-        return int(port)
     return int(port)
 
 def _is_local_address(host_str: str) -> bool:
-    """Check if an address is local or private and should bypass custom DNS."""
-    # Localhost checks
-    if (host_str == 'localhost' or 
-        host_str.startswith('127.') or 
-        host_str == '::1' or 
-        host_str == '0.0.0.0'):
+    """Check if an address is local/private and should bypass custom DNS.
+
+    Returns True for:
+    - 'localhost'
+    - Private/loopback/link-local IP addresses
+    - Simple hostnames without a dot (e.g., 'booklore', 'prowlarr') - likely Docker service names
+    - Hostnames ending in common internal TLDs (.local, .internal, .lan, .home, .docker)
+    """
+    if not host_str:
+        return False
+
+    host_lower = host_str.lower()
+
+    # Check for localhost
+    if host_lower == 'localhost':
         return True
-        
-    # IPv4 private ranges (RFC 1918)
-    if (host_str.startswith('10.') or 
-        (host_str.startswith('172.') and 
-         len(host_str.split('.')) > 1 and 
-         16 <= int(host_str.split('.')[1]) <= 31) or
-        host_str.startswith('192.168.')):
+
+    # Check for simple hostnames (no dot = likely internal Docker/container name)
+    if '.' not in host_str:
         return True
-        
-    # IPv6 private ranges
-    if (host_str.startswith('fc') or 
-        host_str.startswith('fd') or  # Unique local addresses (fc00::/7)
-        host_str.startswith('fe80:')):  # Link-local addresses (fe80::/10)
+
+    # Check for common internal TLDs
+    internal_tlds = ('.local', '.internal', '.lan', '.home', '.docker', '.localdomain')
+    if any(host_lower.endswith(tld) for tld in internal_tlds):
         return True
-    
-    return False
+
+    # Check for private/loopback/link-local IP addresses
+    try:
+        addr = ipaddress.ip_address(host_str)
+        return addr.is_private or addr.is_loopback or addr.is_link_local
+    except ValueError:
+        return False
 
 def _is_ip_address(host_str: str) -> bool:
     """Check if a string is a valid IP address (IPv4 or IPv6)."""
@@ -367,7 +417,7 @@ class DoHResolver:
             response = self.session.get(
                 self.base_url,
                 params=params,
-                proxies=_get_proxies(),
+                proxies=get_proxies(self.base_url),
                 timeout=10  # Increased from 5s to handle slow network conditions
             )
             response.raise_for_status()
@@ -642,8 +692,8 @@ def switch_dns_provider() -> bool:
         name, servers, doh = DNS_PROVIDERS[_current_dns_index]
         CUSTOM_DNS = servers
         DOH_SERVER = doh
-        config.CUSTOM_DNS = servers
-        config.DOH_SERVER = doh
+        app_config.CUSTOM_DNS = servers
+        app_config.DOH_SERVER = doh
 
         logger.warning(f"Switched DNS provider to: {name} (using DoH)")
         _save_state(dns_provider=name)
@@ -672,20 +722,27 @@ def rotate_dns_and_reset_aa() -> bool:
     """
     Switch DNS provider (auto mode) and reset AA URL list to the first entry.
     Returns True if DNS switched; False if no providers left or not in auto mode.
-    
+
     Note: This function can be called during initialization, so we must NOT call
     _ensure_initialized() here to avoid recursive init loops.
     """
     if not rotate_dns_provider():
         return False
     # Reset AA URL to first available auto option if using auto AA
-    global AA_BASE_URL, _current_aa_url_index
-    if AA_BASE_URL == "auto" or AA_BASE_URL in _aa_urls:
+    global _aa_base_url, _current_aa_url_index
+    configured_url = normalize_http_url(
+        app_config.get("AA_BASE_URL", "auto"),
+        default_scheme="https",
+        allow_special=("auto",),
+    )
+    if not configured_url:
+        configured_url = "auto"
+
+    if configured_url == "auto" or _aa_base_url in _aa_urls:
         _current_aa_url_index = 0
-        AA_BASE_URL = _aa_urls[0]
-        config.AA_BASE_URL = AA_BASE_URL
-        logger.info(f"After DNS switch, resetting AA URL to: {AA_BASE_URL}")
-        _save_state(aa_url=AA_BASE_URL)
+        _aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.se"
+        logger.info(f"After DNS switch, resetting AA URL to: {_aa_base_url}")
+        _save_state(aa_url=_aa_base_url)
     return True
 
 def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use_doh: bool | None = None) -> bool:
@@ -715,8 +772,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
             _dns_exhausted_logged = False
             CUSTOM_DNS = []
             DOH_SERVER = ""
-            config.CUSTOM_DNS = []
-            config.DOH_SERVER = ""
+            app_config.CUSTOM_DNS = []
+            app_config.DOH_SERVER = ""
             # Restore original system getaddrinfo
             socket.getaddrinfo = original_getaddrinfo
             logger.info("DNS set to system mode (using OS default resolver)")
@@ -730,8 +787,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
             _dns_exhausted_logged = False
             CUSTOM_DNS = []
             DOH_SERVER = ""
-            config.CUSTOM_DNS = []
-            config.DOH_SERVER = ""
+            app_config.CUSTOM_DNS = []
+            app_config.DOH_SERVER = ""
             logger.info("DNS set to auto mode (system DNS, will rotate on failure with DoH)")
             init_dns_resolvers()
             _notify_dns_rotation("auto", [], "")
@@ -744,8 +801,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
             _current_dns_index = -1  # Not using preset providers
             CUSTOM_DNS = manual_servers
             DOH_SERVER = ""  # No DoH for manual servers
-            config.CUSTOM_DNS = manual_servers
-            config.DOH_SERVER = ""
+            app_config.CUSTOM_DNS = manual_servers
+            app_config.DOH_SERVER = ""
             logger.info(f"DNS set to manual servers: {manual_servers}")
             init_dns_resolvers()
             _notify_dns_rotation("manual", manual_servers, "")
@@ -759,8 +816,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
                 CUSTOM_DNS = servers
                 # Only set DoH server if DoH is enabled
                 DOH_SERVER = doh if doh_enabled else ""
-                config.CUSTOM_DNS = servers
-                config.DOH_SERVER = DOH_SERVER
+                app_config.CUSTOM_DNS = servers
+                app_config.DOH_SERVER = DOH_SERVER
                 doh_status = "DoH enabled" if doh_enabled else "standard DNS"
                 logger.info(f"DNS set to: {name} ({doh_status})")
                 _save_state(dns_provider=name)
@@ -781,15 +838,15 @@ def init_dns_resolvers():
             name, servers, doh = DNS_PROVIDERS[_current_dns_index]
             CUSTOM_DNS = servers
             DOH_SERVER = doh
-            config.CUSTOM_DNS = servers
-            config.DOH_SERVER = doh
+            app_config.CUSTOM_DNS = servers
+            app_config.DOH_SERVER = doh
             logger.info(f"Using DNS provider: {name} (DoH enabled)")
         else:
             CUSTOM_DNS = []
             DOH_SERVER = ""
-            config.CUSTOM_DNS = []
-            config.DOH_SERVER = ""
-            logger.info("Using system DNS (auto mode - will switch on failure)")
+            app_config.CUSTOM_DNS = []
+            app_config.DOH_SERVER = ""
+            logger.debug("Using system DNS (auto mode - will switch on failure)")
             socket.getaddrinfo = cast(Any, create_system_failover_getaddrinfo())
             return
     
@@ -837,35 +894,55 @@ def _looks_like_ip(s: str) -> bool:
     # Simple heuristic: contains only digits, dots, and colons
     return s.replace(".", "").replace(":", "").isdigit()
 
+def _build_aa_urls() -> List[str]:
+    """Build list of available AA URLs from centralized mirror config."""
+    from shelfmark.core.mirrors import get_aa_mirrors
+    return get_aa_mirrors()
+
+
 def _initialize_aa_state() -> None:
     """Restore or probe AA URL state."""
-    global AA_BASE_URL, _current_aa_url_index
-    if AA_BASE_URL == "auto":
+    global _aa_base_url, _current_aa_url_index, _aa_urls
+
+    # Build URL list from config
+    _aa_urls = _build_aa_urls()
+
+    # Get configured base URL from config
+    configured_url = normalize_http_url(
+        app_config.get("AA_BASE_URL", "auto"),
+        default_scheme="https",
+        allow_special=("auto",),
+    )
+    if not configured_url:
+        configured_url = "auto"
+
+    if configured_url == "auto":
         if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls:
             _current_aa_url_index = _aa_urls.index(state['aa_base_url'])
-            AA_BASE_URL = state['aa_base_url']
+            _aa_base_url = state['aa_base_url']
         else:
-            logger.info(f"AA_BASE_URL: auto, checking available urls {_aa_urls}")
+            logger.debug(f"AA_BASE_URL: auto, checking available urls {_aa_urls}")
             for i, url in enumerate(_aa_urls):
                 try:
-                    response = requests.get(url, proxies=_get_proxies(), timeout=3)
+                    response = requests.get(url, proxies=get_proxies(url), timeout=3)
                     if response.status_code == 200:
                         _current_aa_url_index = i
-                        AA_BASE_URL = url
-                        _save_state(aa_url=AA_BASE_URL)
+                        _aa_base_url = url
+                        _save_state(aa_url=_aa_base_url)
                         break
                 except Exception:
                     pass
-            if AA_BASE_URL == "auto":
-                AA_BASE_URL = _aa_urls[0]
+            if not _aa_base_url or _aa_base_url == "auto":
+                _aa_base_url = _aa_urls[0]
                 _current_aa_url_index = 0
-    elif AA_BASE_URL not in _aa_urls:
-        logger.info(f"AA_BASE_URL set to custom value {AA_BASE_URL}; skipping auto-switch")
+    elif configured_url not in _aa_urls:
+        logger.info(f"AA_BASE_URL set to custom value {configured_url}; skipping auto-switch")
+        _aa_base_url = configured_url
     else:
-        _current_aa_url_index = _aa_urls.index(AA_BASE_URL)
+        _current_aa_url_index = _aa_urls.index(configured_url)
+        _aa_base_url = configured_url
 
-    config.AA_BASE_URL = AA_BASE_URL
-    logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
+    logger.info(f"AA_BASE_URL: {_aa_base_url}")
 
 def init_dns(force: bool = False) -> None:
     """Initialize DNS state and resolvers using set_dns_provider() for consistency."""
@@ -939,7 +1016,7 @@ def init(force: bool = False) -> None:
         if _initialized and not force:
             return
         # Do the work first, then set flag to prevent race conditions
-        # where another thread sees _initialized=True but AA_BASE_URL is still "auto"
+        # where another thread sees _initialized=True but _aa_base_url is still empty
         try:
             init_dns(force=force)
             init_aa(force=force)
@@ -952,7 +1029,7 @@ def init(force: bool = False) -> None:
 def get_aa_base_url():
     """Get current AA base URL."""
     _ensure_initialized()
-    return AA_BASE_URL
+    return _aa_base_url
 
 def get_available_aa_urls():
     """Get list of configured AA URLs (copy)."""
@@ -962,14 +1039,13 @@ def get_available_aa_urls():
 def set_aa_url_index(new_index: int) -> bool:
     """Set AA base URL by index in available list; returns True if applied."""
     _ensure_initialized()
-    global AA_BASE_URL, _current_aa_url_index
+    global _aa_base_url, _current_aa_url_index
     if new_index < 0 or new_index >= len(_aa_urls):
         return False
     _current_aa_url_index = new_index
-    AA_BASE_URL = _aa_urls[_current_aa_url_index]
-    config.AA_BASE_URL = AA_BASE_URL
-    logger.info(f"Set AA URL to: {AA_BASE_URL}")
-    _save_state(aa_url=AA_BASE_URL)
+    _aa_base_url = _aa_urls[_current_aa_url_index]
+    logger.info(f"Set AA URL to: {_aa_base_url}")
+    _save_state(aa_url=_aa_base_url)
     return True
 
 class AAMirrorSelector:

shelfmark/download/orchestrator.py

@@ -1,29 +1,11 @@
 """Download queue orchestration and worker management.
 
-## Download Architecture
-
-All downloads follow a two-stage process:
-
-1. **Staging (TMP_DIR)**: Handlers download/copy files to a temp staging area.
-   - Direct downloads: Downloaded directly to staging
-   - Torrent downloads: Copied from torrent client's completed folder to staging
-   - NZB downloads: Moved from NZB client's completed folder to staging
-
-2. **Ingest (INGEST_DIR)**: Orchestrator moves staged files to the final location.
-   - Archive extraction (RAR/ZIP) happens here
-   - Custom scripts run here
-   - Final move to ingest folder
-
-This ensures:
-- Handlers don't need to know about ingest folder logic
-- Archive handling works uniformly for all sources
-- Single point of control for what enters the ingest folder
+Two-stage architecture: handlers stage to TMP_DIR, orchestrator moves to INGEST_DIR
+with archive extraction and custom script support.
 """
 
 import os
 import random
-import shutil
-import subprocess
 import threading
 import time
 from concurrent.futures import Future, ThreadPoolExecutor
@@ -31,80 +13,36 @@ from pathlib import Path
 from threading import Event, Lock
 from typing import Any, Dict, List, Optional, Tuple
 
-from cwa_book_downloader.release_sources import direct_download
-from cwa_book_downloader.release_sources.direct_download import SearchUnavailable
-from cwa_book_downloader.core.config import config
-from cwa_book_downloader.config.env import TMP_DIR, DOWNLOAD_PATHS, INGEST_DIR
-from cwa_book_downloader.download.archive import is_archive, process_archive
-from cwa_book_downloader.release_sources import get_handler, get_source_display_name
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters
-from cwa_book_downloader.core.queue import book_queue
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters, SearchMode
+from shelfmark.core.queue import book_queue
+from shelfmark.core.utils import transform_cover_url
+from shelfmark.download.postprocess.pipeline import is_torrent_source, safe_cleanup_path
+from shelfmark.download.postprocess.router import post_process_download
+from shelfmark.release_sources import direct_download, get_handler, get_source_display_name
+from shelfmark.release_sources.direct_download import SearchUnavailable
 
 logger = setup_logger(__name__)
 
 
 # =============================================================================
-# Staging Directory Helpers
+# Task Download and Processing
 # =============================================================================
-# Handlers should use these to get paths in the staging area.
-# The orchestrator handles moving staged files to the ingest folder.
+#
+# Post-download processing (staging, extraction, transfers, cleanup) lives in
+# `shelfmark.download.postprocess`.
 
-def get_staging_dir() -> Path:
-    """Get the staging directory for downloads.
-
-    All handlers should stage their downloads here. The orchestrator
-    handles moving staged files to the final ingest location.
-    """
-    TMP_DIR.mkdir(parents=True, exist_ok=True)
-    return TMP_DIR
-
-
-def get_staging_path(task_id: str, extension: str) -> Path:
-    """Get a staging path for a download.
-
-    Args:
-        task_id: Unique task identifier
-        extension: File extension (e.g., 'epub', 'zip')
-
-    Returns:
-        Path in staging directory for this download
-    """
-    staging_dir = get_staging_dir()
-    return staging_dir / f"{task_id}.{extension.lstrip('.')}"
-
-
-def stage_file(source_path: Path, task_id: str, copy: bool = False) -> Path:
-    """Stage a file for ingest processing.
-
-    Use this when a download client has completed a download and the file
-    needs to be staged for orchestrator processing.
-
-    Args:
-        source_path: Path to the completed download
-        task_id: Unique task identifier
-        copy: If True, copy the file (for torrents). If False, move it.
-
-    Returns:
-        Path to the staged file
-    """
-    staging_dir = get_staging_dir()
-    staged_path = staging_dir / f"{task_id}{source_path.suffix}"
-
-    if copy:
-        shutil.copy2(str(source_path), str(staged_path))
-        logger.debug(f"Copied to staging: {source_path} -> {staged_path}")
-    else:
-        shutil.move(str(source_path), str(staged_path))
-        logger.debug(f"Moved to staging: {source_path} -> {staged_path}")
-
-    return staged_path
 
 # WebSocket manager (initialized by app.py)
+# Track whether WebSocket is available for status reporting
+WEBSOCKET_AVAILABLE = True
 try:
-    from cwa_book_downloader.api.websocket import ws_manager
+    from shelfmark.api.websocket import ws_manager
 except ImportError:
+    logger.error("WebSocket unavailable - real-time updates disabled")
     ws_manager = None
+    WEBSOCKET_AVAILABLE = False
 
 # Progress update throttling - track last broadcast time per book
 _progress_last_broadcast: Dict[str, float] = {}
@@ -115,15 +53,7 @@ _last_activity: Dict[str, float] = {}
 STALL_TIMEOUT = 300  # 5 minutes without progress/status update = stalled
 
 def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
-    """Search for books matching the query.
-    
-    Args:
-        query: Search term
-        filters: Search filters object
-        
-    Returns:
-        List[Dict]: List of book information dictionaries
-    """
+    """Search for books matching the query."""
     try:
         books = direct_download.search_books(query, filters)
         return [_book_info_to_dict(book) for book in books]
@@ -134,17 +64,7 @@ def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
         raise
 
 def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
-    """Get detailed information for a specific book.
-
-    Args:
-        book_id: Book identifier
-
-    Returns:
-        Optional[Dict]: Book information dictionary if found, None if not found
-
-    Raises:
-        Exception: If there's an error fetching the book info
-    """
+    """Get detailed information for a specific book."""
     try:
         book = direct_download.get_book_info(book_id)
         return _book_info_to_dict(book)
@@ -152,26 +72,14 @@ def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
         logger.error_trace(f"Error getting book info: {e}")
         raise
 
-def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> bool:
-    """Add a book to the download queue with specified priority.
-
-    Fetches display info and creates a DownloadTask. The handler will fetch
-    the full book details (including download URLs) when processing.
-
-    Args:
-        book_id: Book identifier (e.g., AA MD5 hash)
-        priority: Priority level (lower number = higher priority)
-        source: Release source handler to use (default: direct_download)
-
-    Returns:
-        bool: True if book was successfully queued
-    """
+def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> Tuple[bool, Optional[str]]:
+    """Add a book to the download queue. Returns (success, error_message)."""
     try:
-        # Fetch book info for display purposes
-        book_info = direct_download.get_book_info(book_id)
+        book_info = direct_download.get_book_info(book_id, fetch_download_count=False)
         if not book_info:
-            logger.warning(f"Could not fetch book info for {book_id}")
-            return False
+            error_msg = f"Could not fetch book info for {book_id}"
+            logger.warning(error_msg)
+            return False, error_msg
 
         # Create a source-agnostic download task
         task = DownloadTask(
@@ -183,12 +91,13 @@ def queue_book(book_id: str, priority: int = 0, source: str = "direct_download")
             size=book_info.size,
             preview=book_info.preview,
             content_type=book_info.content,
+            search_mode=SearchMode.DIRECT,
             priority=priority,
         )
 
         if not book_queue.add(task):
             logger.info(f"Book already in queue: {book_info.title}")
-            return False
+            return False, "Book is already in the download queue"
 
         logger.info(f"Book queued with priority {priority}: {book_info.title}")
 
@@ -196,53 +105,55 @@ def queue_book(book_id: str, priority: int = 0, source: str = "direct_download")
         if ws_manager:
             ws_manager.broadcast_status_update(queue_status())
 
-        return True
+        return True, None
+    except SearchUnavailable as e:
+        error_msg = f"Search service unavailable: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
     except Exception as e:
-        logger.error_trace(f"Error queueing book: {e}")
-        return False
+        error_msg = f"Error queueing book: {e}"
+        logger.error_trace(error_msg)
+        return False, error_msg
 
 
-def queue_release(release_data: dict, priority: int = 0) -> bool:
-    """Add a release to the download queue.
-
-    This is used when downloading from the ReleaseModal where we already have
-    all the release data from the search - no need to re-fetch.
-
-    Creates a DownloadTask directly from the release data. The handler will
-    fetch full details when processing.
-
-    Args:
-        release_data: Release dictionary with source, source_id, title, format, etc.
-        priority: Priority level (lower number = higher priority)
-
-    Returns:
-        bool: True if release was successfully queued
-    """
+def queue_release(release_data: dict, priority: int = 0) -> Tuple[bool, Optional[str]]:
+    """Add a release to the download queue. Returns (success, error_message)."""
     try:
         source = release_data.get('source', 'direct_download')
         extra = release_data.get('extra', {})
 
-        # Get author, preview, and content_type from top-level (preferred) or extra (fallback)
+        # Get author, year, preview, and content_type from top-level (preferred) or extra (fallback)
         author = release_data.get('author') or extra.get('author')
+        year = release_data.get('year') or extra.get('year')
         preview = release_data.get('preview') or extra.get('preview')
         content_type = release_data.get('content_type') or extra.get('content_type')
 
+        # Get series info for library naming templates
+        series_name = release_data.get('series_name') or extra.get('series_name')
+        series_position = release_data.get('series_position') or extra.get('series_position')
+        subtitle = release_data.get('subtitle') or extra.get('subtitle')
+
         # Create a source-agnostic download task from release data
         task = DownloadTask(
             task_id=release_data['source_id'],
             source=source,
             title=release_data.get('title', 'Unknown'),
             author=author,
+            year=year,
             format=release_data.get('format'),
             size=release_data.get('size'),
             preview=preview,
             content_type=content_type,
+            series_name=series_name,
+            series_position=series_position,
+            subtitle=subtitle,
+            search_mode=SearchMode.UNIVERSAL,
             priority=priority,
         )
 
         if not book_queue.add(task):
             logger.info(f"Release already in queue: {task.title}")
-            return False
+            return False, "Release is already in the download queue"
 
         logger.info(f"Release queued with priority {priority}: {task.title}")
 
@@ -250,28 +161,29 @@ def queue_release(release_data: dict, priority: int = 0) -> bool:
         if ws_manager:
             ws_manager.broadcast_status_update(queue_status())
 
-        return True
+        return True, None
 
     except ValueError as e:
         # Handler not found for this source
-        logger.warning(f"Unknown release source: {e}")
-        return False
+        error_msg = f"Unknown release source: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except KeyError as e:
+        error_msg = f"Missing required field in release data: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
     except Exception as e:
-        logger.error_trace(f"Error queueing release: {e}")
-        return False
+        error_msg = f"Error queueing release: {e}"
+        logger.error_trace(error_msg)
+        return False, error_msg
 
 def queue_status() -> Dict[str, Dict[str, Any]]:
-    """Get current status of the download queue.
-
-    Returns:
-        Dict: Queue status organized by status type with serialized task data
-    """
+    """Get current status of the download queue."""
     status = book_queue.get_status()
     for _, tasks in status.items():
         for _, task in tasks.items():
-            if task.download_path:
-                if not os.path.exists(task.download_path):
-                    task.download_path = None
+            if task.download_path and not os.path.exists(task.download_path):
+                task.download_path = None
 
     # Convert Enum keys to strings and DownloadTask objects to dicts for JSON serialization
     return {
@@ -283,14 +195,7 @@ def queue_status() -> Dict[str, Dict[str, Any]]:
     }
 
 def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]]:
-    """Get downloaded file data for a specific task.
-
-    Args:
-        task_id: Task identifier
-
-    Returns:
-        Tuple[Optional[bytes], Optional[DownloadTask]]: File data if available, and the task
-    """
+    """Get downloaded file data for a specific task."""
     task = None
     try:
         task = book_queue.get_task(task_id)
@@ -310,45 +215,23 @@ def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]
         return None, task
 
 def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
-    """Convert BookInfo object to dictionary representation.
-
-    Transforms external preview URLs to local proxy URLs when cover caching is enabled.
-    """
-    import base64
-    from cwa_book_downloader.config.env import is_covers_cache_enabled
-
+    """Convert BookInfo to dict, transforming cover URLs for caching."""
     result = {
         key: value for key, value in book.__dict__.items()
         if value is not None
     }
 
     # Transform external preview URLs to local proxy URLs
-    # Skip if already a local URL (starts with /)
-    if result.get('preview') and is_covers_cache_enabled() and not result['preview'].startswith('/'):
-        original_url = result['preview']
-        encoded_url = base64.urlsafe_b64encode(original_url.encode()).decode()
-        result['preview'] = f"/api/covers/{book.id}?url={encoded_url}"
+    if result.get('preview'):
+        result['preview'] = transform_cover_url(result['preview'], book.id)
 
     return result
 
 
 def _task_to_dict(task: DownloadTask) -> Dict[str, Any]:
-    """Convert DownloadTask object to dictionary representation.
-
-    Maps DownloadTask fields to the format expected by the frontend,
-    maintaining compatibility with the previous BookInfo-based format.
-    Transforms external preview URLs to local proxy URLs when cover caching is enabled.
-    """
-    import base64
-    from cwa_book_downloader.config.env import is_covers_cache_enabled
-
-    preview = task.preview
-
+    """Convert DownloadTask to dict for frontend, transforming cover URLs."""
     # Transform external preview URLs to local proxy URLs
-    # Skip if already a local URL (starts with /)
-    if preview and is_covers_cache_enabled() and not preview.startswith('/'):
-        encoded_url = base64.urlsafe_b64encode(preview.encode()).decode()
-        preview = f"/api/covers/{task.task_id}?url={encoded_url}"
+    preview = transform_cover_url(task.preview, task.task_id)
 
     return {
         'id': task.task_id,
@@ -370,33 +253,31 @@ def _task_to_dict(task: DownloadTask) -> Dict[str, Any]:
 
 
 def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
-    """Download a task with cancellation support.
-
-    Delegates to the appropriate handler based on the task's source.
-    Handlers return a temp file path, orchestrator handles post-processing
-    (archive extraction, moving to ingest) uniformly for all sources.
-
-    Args:
-        task_id: Task identifier
-        cancel_flag: Threading event to signal cancellation
-
-    Returns:
-        str: Path to the downloaded file if successful, None otherwise
-    """
+    """Download a task via appropriate handler, then post-process to ingest."""
     try:
         # Check for cancellation before starting
         if cancel_flag.is_set():
-            logger.info(f"Download cancelled before starting: {task_id}")
+            logger.info("Task %s: cancelled before starting", task_id)
             return None
 
         task = book_queue.get_task(task_id)
         if not task:
-            logger.error(f"Task not found in queue: {task_id}")
+            logger.error("Task not found in queue: %s", task_id)
             return None
 
-        # Create callbacks that update the orchestrator's tracking
-        progress_callback = lambda progress: update_download_progress(task_id, progress)
-        status_callback = lambda status, message=None: update_download_status(task_id, status, message)
+        title_label = task.title or "Unknown title"
+        logger.info(
+            "Task %s: starting download (%s) - %s",
+            task_id,
+            get_source_display_name(task.source),
+            title_label,
+        )
+
+        def progress_callback(progress: float) -> None:
+            update_download_progress(task_id, progress)
+
+        def status_callback(status: str, message: Optional[str] = None) -> None:
+            update_download_status(task_id, status, message)
 
         # Get the download handler based on the task's source
         handler = get_handler(task.source)
@@ -418,130 +299,58 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
 
         # Check cancellation before post-processing
         if cancel_flag.is_set():
-            logger.info(f"Download cancelled before post-processing: {task_id}")
-            temp_file.unlink(missing_ok=True)
+            logger.info("Task %s: cancelled before post-processing", task_id)
+            if not is_torrent_source(temp_file, task):
+                safe_cleanup_path(temp_file, task)
             return None
 
-        # Post-processing: archive extraction or direct move to ingest
-        return _post_process_download(
-            temp_file, task, cancel_flag, status_callback
-        )
+        logger.info("Task %s: download finished; starting post-processing", task_id)
+        logger.debug("Task %s: post-processing input path: %s", task_id, temp_file)
+
+        # Post-processing: output routing + file processing pipeline
+        result = post_process_download(temp_file, task, cancel_flag, status_callback)
+
+        if cancel_flag.is_set():
+            logger.info("Task %s: post-processing cancelled", task_id)
+        elif result:
+            logger.info("Task %s: post-processing complete", task_id)
+            logger.debug("Task %s: post-processing result: %s", task_id, result)
+        else:
+            logger.warning("Task %s: post-processing failed", task_id)
+
+        try:
+            handler.post_process_cleanup(task, success=bool(result))
+        except Exception as e:
+            logger.warning("Post-processing cleanup hook failed for %s: %s", task_id, e)
+
+        return result
 
     except Exception as e:
         if cancel_flag.is_set():
-            logger.info(f"Download cancelled during error handling: {task_id}")
+            logger.info("Task %s: cancelled during error handling", task_id)
         else:
-            logger.error_trace(f"Error downloading: {e}")
+            logger.error_trace("Task %s: error downloading: %s", task_id, e)
+            # Update task status so user sees the failure
+            task = book_queue.get_task(task_id)
+            if task:
+                book_queue.update_status(task_id, QueueStatus.ERROR)
+                # Check for known misconfiguration from earlier versions
+                if isinstance(e, PermissionError) and "/cwa-book-ingest" in str(e):
+                    book_queue.update_status_message(
+                        task_id,
+                        "Destination misconfigured. Go to Settings → Downloads to update."
+                    )
+                else:
+                    if isinstance(e, PermissionError):
+                        book_queue.update_status_message(task_id, f"Permission denied: {e}")
+                    else:
+                        book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}")
         return None
 
 
-def _post_process_download(
-    temp_file: Path,
-    task: DownloadTask,
-    cancel_flag: Event,
-    status_callback,
-) -> Optional[str]:
-    """Post-process a downloaded file: handle archives and move to ingest.
-
-    This runs uniformly for all download sources, ensuring consistent behavior.
-
-    Args:
-        temp_file: Path to downloaded file in temp directory
-        task: Download task with metadata
-        cancel_flag: Cancellation event
-        status_callback: Callback for status updates
-
-    Returns:
-        Final path in ingest directory, or None on failure
-    """
-    # Route to content-type-specific ingest directory if configured
-    content_type = task.content_type.lower() if task.content_type else None
-    ingest_dir = DOWNLOAD_PATHS.get(content_type, INGEST_DIR)
-    if content_type and ingest_dir != INGEST_DIR:
-        logger.debug(f"Routing content type '{content_type}' to {ingest_dir}")
-    os.makedirs(ingest_dir, exist_ok=True)
-
-    # Handle archive extraction (RAR/ZIP)
-    if is_archive(temp_file):
-        logger.info(f"Archive detected, extracting: {temp_file.name}")
-        status_callback("resolving", "Extracting archive...")
-
-        result = process_archive(
-            archive_path=temp_file,
-            temp_dir=TMP_DIR,
-            ingest_dir=ingest_dir,
-            archive_id=task.task_id,
-        )
-
-        if result.success:
-            status_callback("complete", result.message)
-            return str(result.final_paths[0])
-        else:
-            status_callback("error", result.error)
-            return None
-
-    # Non-archive: run custom script if configured, then move to ingest
-    if config.CUSTOM_SCRIPT:
-        logger.info(f"Running custom script: {config.CUSTOM_SCRIPT}")
-        subprocess.run([config.CUSTOM_SCRIPT, str(temp_file)])
-
-    # Check cancellation before final move
-    if cancel_flag.is_set():
-        logger.info(f"Download cancelled before ingest: {task.task_id}")
-        temp_file.unlink(missing_ok=True)
-        return None
-
-    # Generate filename and move to ingest
-    filename = task.get_filename()
-    if not filename:
-        filename = f"{task.task_id}.{task.format or 'bin'}"
-
-    final_path = ingest_dir / filename
-
-    # Handle duplicate filenames
-    if final_path.exists():
-        base = final_path.stem
-        ext = final_path.suffix
-        counter = 1
-        while final_path.exists():
-            final_path = ingest_dir / f"{base}_{counter}{ext}"
-            counter += 1
-        logger.info(f"File already exists, saving as: {final_path.name}")
-
-    # Use intermediate .crdownload file for atomic move
-    intermediate_path = ingest_dir / f"{task.task_id}.crdownload"
-
-    try:
-        shutil.move(str(temp_file), str(intermediate_path))
-    except Exception as e:
-        logger.debug(f"Error moving file: {e}, trying copy instead")
-        try:
-            shutil.copyfile(str(temp_file), str(intermediate_path))
-            temp_file.unlink(missing_ok=True)
-        except Exception as e2:
-            logger.error(f"Failed to move/copy file to ingest: {e2}")
-            return None
-
-    # Final cancellation check
-    if cancel_flag.is_set():
-        logger.info(f"Download cancelled before final rename: {task.task_id}")
-        intermediate_path.unlink(missing_ok=True)
-        return None
-
-    os.rename(str(intermediate_path), str(final_path))
-    logger.info(f"Download completed: {final_path.name}")
-
-    return str(final_path)
 
 def update_download_progress(book_id: str, progress: float) -> None:
-    """Update download progress with throttled WebSocket broadcasts.
-
-    Progress is always stored in the queue, but WebSocket broadcasts are
-    throttled to avoid flooding clients with updates. Broadcasts occur:
-    - At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds
-    - Always at 0% (start) and 100% (complete)
-    - On significant progress jumps (>10%)
-    """
+    """Update download progress with throttled WebSocket broadcasts."""
     book_queue.update_progress(book_id, progress)
 
     # Track activity for stall detection
@@ -576,13 +385,7 @@ def update_download_progress(book_id: str, progress: float) -> None:
             ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
 
 def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
-    """Update download status with optional detailed message.
-    
-    Args:
-        book_id: Book identifier
-        status: Status string (e.g., 'resolving', 'downloading')
-        message: Optional detailed status message for UI display
-    """
+    """Update download status with optional message for UI display."""
     # Map string status to QueueStatus enum
     status_map = {
         'queued': QueueStatus.QUEUED,
@@ -612,14 +415,7 @@ def update_download_status(book_id: str, status: str, message: Optional[str] = N
             ws_manager.broadcast_status_update(queue_status())
 
 def cancel_download(book_id: str) -> bool:
-    """Cancel a download.
-    
-    Args:
-        book_id: Book identifier to cancel
-        
-    Returns:
-        bool: True if cancellation was successful
-    """
+    """Cancel a download."""
     result = book_queue.cancel_download(book_id)
     
     # Broadcast status update via WebSocket
@@ -629,29 +425,14 @@ def cancel_download(book_id: str) -> bool:
     return result
 
 def set_book_priority(book_id: str, priority: int) -> bool:
-    """Set priority for a queued book.
-    
-    Args:
-        book_id: Book identifier
-        priority: New priority level (lower = higher priority)
-        
-    Returns:
-        bool: True if priority was successfully changed
-    """
+    """Set priority for a queued book (lower = higher priority)."""
     return book_queue.set_priority(book_id, priority)
 
 def reorder_queue(book_priorities: Dict[str, int]) -> bool:
-    """Bulk reorder queue.
-    
-    Args:
-        book_priorities: Dict mapping book_id to new priority
-        
-    Returns:
-        bool: True if reordering was successful
-    """
+    """Bulk reorder queue by mapping book_id to new priority."""
     return book_queue.reorder_queue(book_priorities)
 
-def get_queue_order() -> List[Dict[str, any]]:
+def get_queue_order() -> List[Dict[str, Any]]:
     """Get current queue order for display."""
     return book_queue.get_queue_order()
 
@@ -776,11 +557,7 @@ _started = False
 
 
 def start() -> None:
-    """Start the download coordinator thread.
-
-    This should be called once during application startup.
-    Calling multiple times is safe - subsequent calls are no-ops.
-    """
+    """Start the download coordinator thread. Safe to call multiple times."""
     global _coordinator_thread, _started
 
     if _started:

shelfmark/download/outputs/__init__.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Callable, Optional
+
+from shelfmark.core.models import DownloadTask
+
+StatusCallback = Callable[[str, Optional[str]], None]
+OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback], Optional[str]]
+
+
+@dataclass(frozen=True)
+class OutputRegistration:
+    mode: str
+    supports_task: Callable[[DownloadTask], bool]
+    handler: OutputHandler
+    priority: int = 0
+
+
+_OUTPUT_REGISTRY: list[OutputRegistration] = []
+_OUTPUTS_LOADED = False
+
+
+def register_output(
+    mode: str,
+    supports_task: Callable[[DownloadTask], bool],
+    priority: int = 0,
+) -> Callable[[OutputHandler], OutputHandler]:
+    def decorator(handler: OutputHandler) -> OutputHandler:
+        _OUTPUT_REGISTRY.append(
+            OutputRegistration(
+                mode=mode,
+                supports_task=supports_task,
+                handler=handler,
+                priority=priority,
+            )
+        )
+        _OUTPUT_REGISTRY.sort(key=lambda entry: entry.priority, reverse=True)
+        return handler
+
+    return decorator
+
+
+def load_output_handlers() -> None:
+    global _OUTPUTS_LOADED
+    if _OUTPUTS_LOADED:
+        return
+
+    from . import booklore  # noqa: F401
+    from . import folder  # noqa: F401
+
+    _OUTPUTS_LOADED = True
+
+
+def resolve_output_handler(task: DownloadTask) -> Optional[OutputRegistration]:
+    load_output_handlers()
+    for entry in _OUTPUT_REGISTRY:
+        if entry.supports_task(task):
+            return entry
+    return None

shelfmark/download/outputs/booklore.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Any, Dict, List, Mapping, Optional
+
+import requests
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.outputs import register_output
+from shelfmark.download.staging import STAGE_MOVE, STAGE_NONE, build_staging_dir
+
+logger = setup_logger(__name__)
+
+BOOKLORE_OUTPUT_MODE = "booklore"
+BOOKLORE_SUPPORTED_EXTENSIONS = {".cb7", ".cbr", ".cbz", ".epub", ".fb2", ".pdf"}
+BOOKLORE_SUPPORTED_FORMATS_LABEL = ", ".join(
+    ext.lstrip(".").upper() for ext in sorted(BOOKLORE_SUPPORTED_EXTENSIONS)
+)
+
+
+class BookloreError(Exception):
+    """Raised when Booklore integration fails."""
+
+
+@dataclass(frozen=True)
+class BookloreConfig:
+    base_url: str
+    username: str
+    password: str
+    library_id: int
+    path_id: int
+    verify_tls: bool = True
+    refresh_after_upload: bool = False
+
+
+def _parse_int(value: Any, label: str) -> int:
+    if value is None or value == "":
+        raise BookloreError(f"{label} is required")
+    try:
+        return int(value)
+    except (TypeError, ValueError) as exc:
+        raise BookloreError(f"{label} must be a number") from exc
+
+
+def build_booklore_config(values: Mapping[str, Any]) -> BookloreConfig:
+    base_url = str(values.get("BOOKLORE_HOST", "")).strip()
+    username = str(values.get("BOOKLORE_USERNAME", "")).strip()
+    password = values.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        raise BookloreError("Booklore URL is required")
+    if not username:
+        raise BookloreError("Booklore username is required")
+    if not password:
+        raise BookloreError("Booklore password is required")
+
+    library_id = _parse_int(values.get("BOOKLORE_LIBRARY_ID"), "Booklore library ID")
+    path_id = _parse_int(values.get("BOOKLORE_PATH_ID"), "Booklore path ID")
+
+    return BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=library_id,
+        path_id=path_id,
+        verify_tls=True,
+        refresh_after_upload=True,  # Always refresh library after upload
+    )
+
+
+def booklore_login(booklore_config: BookloreConfig) -> str:
+    url = f"{booklore_config.base_url}/api/v1/auth/login"
+    payload = {"username": booklore_config.username, "password": booklore_config.password}
+
+    try:
+        response = requests.post(url, json=payload, timeout=30, verify=booklore_config.verify_tls)
+    except requests.exceptions.ConnectionError as exc:
+        raise BookloreError("Could not connect to Booklore") from exc
+    except requests.exceptions.Timeout as exc:
+        raise BookloreError("Booklore connection timed out") from exc
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore login failed: {exc}") from exc
+
+    if response.status_code in {401, 403}:
+        raise BookloreError("Booklore authentication failed")
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        raise BookloreError(f"Booklore login failed ({response.status_code})") from exc
+
+    try:
+        data = response.json()
+    except ValueError as exc:
+        raise BookloreError("Invalid Booklore login response") from exc
+
+    token = data.get("accessToken")
+    if not token:
+        raise BookloreError("Booklore did not return an access token")
+
+    return token
+
+
+def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list[dict[str, Any]]:
+    url = f"{booklore_config.base_url}/api/v1/libraries"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Failed to fetch Booklore libraries: {exc}") from exc
+
+    try:
+        return response.json()
+    except ValueError as exc:
+        raise BookloreError("Invalid Booklore libraries response") from exc
+
+
+def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None:
+    url = f"{booklore_config.base_url}/api/v1/files/upload"
+    headers = {"Authorization": f"Bearer {token}"}
+    params = {"libraryId": booklore_config.library_id, "pathId": booklore_config.path_id}
+
+    response = None
+
+    try:
+        with file_path.open("rb") as handle:
+            response = requests.post(
+                url,
+                headers=headers,
+                params=params,
+                files={"file": (file_path.name, handle)},
+                timeout=60,
+                verify=booklore_config.verify_tls,
+            )
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        message = response.text.strip() if response is not None else ""
+        if message:
+            message = f": {message[:200]}"
+        status_code = response.status_code if response is not None else "unknown"
+        raise BookloreError(f"Booklore upload failed ({status_code}){message}") from exc
+    except requests.exceptions.ConnectionError as exc:
+        raise BookloreError("Could not connect to Booklore") from exc
+    except requests.exceptions.Timeout as exc:
+        raise BookloreError("Booklore upload timed out") from exc
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore upload failed: {exc}") from exc
+
+
+def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None:
+    url = f"{booklore_config.base_url}/api/v1/libraries/{booklore_config.library_id}/refresh"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.put(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore refresh failed: {exc}") from exc
+
+
+def _supports_booklore(task: DownloadTask) -> bool:
+    if check_audiobook(task.content_type):
+        return False
+    return core_config.config.get("BOOKS_OUTPUT_MODE", "folder") == BOOKLORE_OUTPUT_MODE
+
+
+def _get_booklore_settings() -> Dict[str, Any]:
+    return {
+        "BOOKLORE_HOST": core_config.config.get("BOOKLORE_HOST", ""),
+        "BOOKLORE_USERNAME": core_config.config.get("BOOKLORE_USERNAME", ""),
+        "BOOKLORE_PASSWORD": core_config.config.get("BOOKLORE_PASSWORD", ""),
+        "BOOKLORE_LIBRARY_ID": core_config.config.get("BOOKLORE_LIBRARY_ID"),
+        "BOOKLORE_PATH_ID": core_config.config.get("BOOKLORE_PATH_ID"),
+    }
+
+
+def _booklore_format_error(rejected_files: List[Path]) -> str:
+    rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+    rejected_list = ", ".join(rejected_exts)
+    return (
+        f"Booklore does not support {rejected_list}. "
+        f"Supported formats: {BOOKLORE_SUPPORTED_FORMATS_LABEL}"
+    )
+
+
+def _post_process_booklore(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    from shelfmark.download.postprocess.pipeline import (
+        OutputPlan,
+        cleanup_output_staging,
+        is_managed_workspace_path,
+        prepare_output_files,
+    )
+
+    if cancel_flag.is_set():
+        logger.info("Task %s: cancelled before Booklore upload", task.task_id)
+        return None
+
+    try:
+        booklore_config = build_booklore_config(_get_booklore_settings())
+    except BookloreError as e:
+        logger.warning("Task %s: Booklore configuration error: %s", task.task_id, e)
+        status_callback("error", str(e))
+        return None
+
+    status_callback("resolving", "Preparing Booklore upload")
+
+    output_plan = OutputPlan(
+        mode=BOOKLORE_OUTPUT_MODE,
+        stage_action=STAGE_MOVE if is_managed_workspace_path(temp_file) else STAGE_NONE,
+        staging_dir=build_staging_dir("booklore", task.task_id),
+        allow_archive_extraction=True,
+    )
+
+    prepared = prepare_output_files(
+        temp_file,
+        task,
+        BOOKLORE_OUTPUT_MODE,
+        status_callback,
+        output_plan=output_plan,
+    )
+    if not prepared:
+        return None
+
+    logger.debug("Task %s: prepared %d file(s) for Booklore upload", task.task_id, len(prepared.files))
+
+    try:
+        unsupported_files = [
+            file_path
+            for file_path in prepared.files
+            if file_path.suffix.lower() not in BOOKLORE_SUPPORTED_EXTENSIONS
+        ]
+        if unsupported_files:
+            error_message = _booklore_format_error(unsupported_files)
+            logger.warning("Task %s: %s", task.task_id, error_message)
+            status_callback("error", error_message)
+            return None
+
+        token = booklore_login(booklore_config)
+        logger.info("Task %s: uploading %d file(s) to Booklore", task.task_id, len(prepared.files))
+
+        for index, file_path in enumerate(prepared.files, start=1):
+            if cancel_flag.is_set():
+                logger.info("Task %s: cancelled during Booklore upload", task.task_id)
+                return None
+            status_callback("resolving", f"Uploading to Booklore ({index}/{len(prepared.files)})")
+            booklore_upload_file(booklore_config, token, file_path)
+
+        if booklore_config.refresh_after_upload:
+            try:
+                booklore_refresh_library(booklore_config, token)
+            except BookloreError as e:
+                logger.warning("Task %s: Booklore refresh failed: %s", task.task_id, e)
+
+        logger.info("Task %s: uploaded %d file(s) to Booklore", task.task_id, len(prepared.files))
+
+        message = "Uploaded to Booklore"
+        if len(prepared.files) > 1:
+            message = f"Uploaded to Booklore ({len(prepared.files)} files)"
+        status_callback("complete", message)
+        return f"booklore://{task.task_id}"
+
+    except BookloreError as e:
+        logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e)
+        status_callback("error", str(e))
+        return None
+    except Exception as e:
+        logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e)
+        status_callback("error", f"Booklore upload failed: {e}")
+        return None
+    finally:
+        cleanup_output_staging(
+            prepared.output_plan,
+            prepared.working_path,
+            task,
+            prepared.cleanup_paths,
+        )
+
+
+@register_output(BOOKLORE_OUTPUT_MODE, supports_task=_supports_booklore, priority=10)
+def process_booklore_output(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    return _post_process_booklore(temp_file, task, cancel_flag, status_callback)

shelfmark/download/outputs/folder.py

@@ -0,0 +1,310 @@
+from __future__ import annotations
+
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Any, Optional, List
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.archive import is_archive
+from shelfmark.download.outputs import register_output
+from shelfmark.download.staging import StageAction, STAGE_NONE
+
+logger = setup_logger(__name__)
+
+FOLDER_OUTPUT_MODE = "folder"
+
+
+def _resolve_custom_script_target(target_path: Path, destination: Path, path_mode: str) -> Path:
+    mode = (path_mode or "absolute").strip().lower()
+    if mode != "relative":
+        return target_path
+
+    try:
+        return target_path.relative_to(destination)
+    except ValueError:
+        if target_path.is_absolute():
+            return Path(target_path.name)
+        return target_path
+
+
+@dataclass(frozen=True)
+class _ProcessingPlan:
+    destination: Path
+    organization_mode: str
+    use_hardlink: bool
+    allow_archive_extraction: bool
+    stage_action: StageAction
+    staging_dir: Path
+    hardlink_source: Optional[Path]
+    output_mode: str = FOLDER_OUTPUT_MODE
+
+
+def _supports_folder_output(task: DownloadTask) -> bool:
+    if check_audiobook(task.content_type):
+        return True
+    return core_config.config.get("BOOKS_OUTPUT_MODE", FOLDER_OUTPUT_MODE) == FOLDER_OUTPUT_MODE
+
+
+def _build_processing_plan(
+    temp_file: Path,
+    task: DownloadTask,
+    status_callback,
+) -> Optional[_ProcessingPlan]:
+    from shelfmark.download.postprocess.pipeline import (
+        build_output_plan,
+        get_final_destination,
+        validate_destination,
+    )
+    from shelfmark.download.postprocess.policy import get_file_organization
+
+    is_audiobook = check_audiobook(task.content_type)
+    organization_mode = get_file_organization(is_audiobook)
+    destination = get_final_destination(task)
+
+    if not validate_destination(destination, status_callback):
+        return None
+
+    output_plan = build_output_plan(
+        temp_file,
+        task,
+        output_mode=FOLDER_OUTPUT_MODE,
+        destination=destination,
+        status_callback=status_callback,
+    )
+    if not output_plan.transfer_plan:
+        return None
+
+    transfer_plan = output_plan.transfer_plan
+    hardlink_source = transfer_plan.source_path if transfer_plan.use_hardlink else None
+
+    return _ProcessingPlan(
+        destination=destination,
+        organization_mode=organization_mode,
+        use_hardlink=transfer_plan.use_hardlink,
+        allow_archive_extraction=transfer_plan.allow_archive_extraction,
+        stage_action=output_plan.stage_action,
+        staging_dir=output_plan.staging_dir,
+        hardlink_source=hardlink_source,
+    )
+
+
+@register_output(FOLDER_OUTPUT_MODE, supports_task=_supports_folder_output, priority=0)
+def process_folder_output(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    """Post-process download to the configured folder destination."""
+    from shelfmark.download.postprocess.pipeline import (
+        cleanup_output_staging,
+        is_torrent_source,
+        log_plan_steps,
+        prepare_output_files,
+        record_step,
+        safe_cleanup_path,
+        transfer_book_files,
+    )
+
+    plan = _build_processing_plan(temp_file, task, status_callback)
+    if not plan:
+        return None
+
+    logger.debug(
+        "Processing plan for task %s: mode=%s destination=%s hardlink=%s stage_action=%s extract_archives=%s",
+        task.task_id,
+        plan.organization_mode,
+        plan.destination,
+        plan.use_hardlink,
+        plan.stage_action,
+        plan.allow_archive_extraction,
+    )
+
+    prepared = prepare_output_files(
+        temp_file,
+        task,
+        output_mode=plan.output_mode,
+        status_callback=status_callback,
+        destination=plan.destination,
+    )
+    if not prepared:
+        return None
+
+    steps: List[Any] = []
+    if prepared.output_plan.stage_action != STAGE_NONE:
+        step_name = f"stage_{prepared.output_plan.stage_action}"
+        record_step(steps, step_name, source=str(temp_file), dest=str(prepared.output_plan.staging_dir))
+
+    def run_custom_script(script_path: str, target_path: Path, phase: str) -> bool:
+        path_mode = core_config.config.get("CUSTOM_SCRIPT_PATH_MODE", "absolute")
+        script_target = _resolve_custom_script_target(target_path, plan.destination, path_mode)
+        env = {
+            **os.environ,
+            "SHELFMARK_CUSTOM_SCRIPT_TARGET": str(target_path),
+            "SHELFMARK_CUSTOM_SCRIPT_RELATIVE": str(_resolve_custom_script_target(target_path, plan.destination, "relative")),
+            "SHELFMARK_CUSTOM_SCRIPT_DESTINATION": str(plan.destination),
+            "SHELFMARK_CUSTOM_SCRIPT_MODE": str(path_mode),
+            "SHELFMARK_CUSTOM_SCRIPT_PHASE": phase,
+        }
+        record_step(
+            steps,
+            "custom_script",
+            script=str(script_path),
+            target=str(script_target),
+            target_abs=str(target_path),
+            mode=str(path_mode),
+            phase=phase,
+        )
+        log_plan_steps(task.task_id, steps)
+        logger.info(
+            "Task %s: running custom script %s on %s (%s)",
+            task.task_id,
+            script_path,
+            script_target,
+            phase,
+        )
+        try:
+            result = subprocess.run(
+                [script_path, str(script_target)],
+                check=True,
+                timeout=300,  # 5 minute timeout
+                capture_output=True,
+                text=True,
+                env=env,
+            )
+            if result.stdout:
+                logger.debug("Task %s: custom script stdout: %s", task.task_id, result.stdout.strip())
+            return True
+        except FileNotFoundError:
+            logger.error("Task %s: custom script not found: %s", task.task_id, script_path)
+            status_callback("error", f"Custom script not found: {script_path}")
+            return False
+        except PermissionError:
+            logger.error("Task %s: custom script not executable: %s", task.task_id, script_path)
+            status_callback("error", f"Custom script not executable: {script_path}")
+            return False
+        except subprocess.TimeoutExpired:
+            logger.error("Task %s: custom script timed out after 300s: %s", task.task_id, script_path)
+            status_callback("error", "Custom script timed out")
+            return False
+        except subprocess.CalledProcessError as e:
+            stderr = e.stderr.strip() if e.stderr else "No error output"
+            logger.error(
+                "Task %s: custom script failed (exit code %s): %s",
+                task.task_id,
+                e.returncode,
+                stderr,
+            )
+            status_callback("error", f"Custom script failed: {stderr[:100]}")
+            return False
+
+    # Custom script is run post-transfer (see below).
+
+    # If we staged a copy into TMP_DIR (e.g. for custom script), transfer from the staged
+    # path and disable hardlinking for this transfer.
+    use_hardlink = plan.use_hardlink and prepared.output_plan.stage_action == STAGE_NONE
+    source_path = plan.hardlink_source if use_hardlink and plan.hardlink_source else prepared.working_path
+    is_torrent = is_torrent_source(source_path, task)
+
+    usenet_action = core_config.config.get("PROWLARR_USENET_ACTION", "move")
+    is_usenet = task.source == "prowlarr" and not task.original_download_path
+
+    # For external usenet downloads, always copy from the client path.
+    # "Move" is implemented as a client-side cleanup after import.
+    preserve_source = is_usenet
+
+    copy_for_label = is_torrent or preserve_source or prepared.output_plan.stage_action != STAGE_NONE
+
+    if cancel_flag.is_set():
+        logger.info("Task %s: cancelled before final transfer", task.task_id)
+        cleanup_output_staging(
+            prepared.output_plan,
+            prepared.working_path,
+            task,
+            prepared.cleanup_paths,
+        )
+        return None
+
+    if use_hardlink:
+        op_label = "Hardlinking"
+    elif is_usenet and usenet_action == "move" and prepared.output_plan.stage_action == STAGE_NONE:
+        # Presented as a move, but implemented as copy + client cleanup.
+        op_label = "Moving"
+    elif copy_for_label:
+        op_label = "Copying"
+    else:
+        op_label = "Moving"
+
+    status_callback("resolving", f"{op_label} file")
+    record_step(
+        steps,
+        "transfer",
+        op=op_label.lower(),
+        source=str(source_path),
+        dest=str(plan.destination),
+        hardlink=use_hardlink,
+        torrent=copy_for_label,
+    )
+    if prepared.output_plan.stage_action != STAGE_NONE:
+        record_step(steps, "cleanup_staging", path=str(prepared.working_path))
+    log_plan_steps(task.task_id, steps)
+
+    final_paths, error = transfer_book_files(
+        prepared.files,
+        destination=plan.destination,
+        task=task,
+        use_hardlink=use_hardlink,
+        is_torrent=is_torrent,
+        preserve_source=preserve_source,
+        organization_mode=plan.organization_mode,
+    )
+
+    if error:
+        logger.warning("Task %s: transfer failed: %s", task.task_id, error)
+        status_callback("error", error)
+        return None
+
+    logger.info(
+        "Task %s: transferred %d file(s) to %s (%s)",
+        task.task_id,
+        len(final_paths),
+        plan.destination,
+        op_label.lower(),
+    )
+
+    # Run custom script once per successful task, after transfer.
+    if core_config.config.CUSTOM_SCRIPT:
+        if len(final_paths) == 1:
+            target_path = final_paths[0]
+        else:
+            try:
+                target_path = Path(os.path.commonpath([str(p.parent) for p in final_paths]))
+            except ValueError:
+                target_path = plan.destination
+
+        if not run_custom_script(core_config.config.CUSTOM_SCRIPT, target_path, phase="post_transfer"):
+            cleanup_output_staging(
+                prepared.output_plan,
+                prepared.working_path,
+                task,
+                prepared.cleanup_paths,
+            )
+            return None
+
+    cleanup_output_staging(
+        prepared.output_plan,
+        prepared.working_path,
+        task,
+        prepared.cleanup_paths,
+    )
+
+    message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
+    status_callback("complete", message)
+
+    return str(final_paths[0])

shelfmark/download/permissions_debug.py

@@ -0,0 +1,125 @@
+"""Permission/ownership diagnostics for filesystem operations.
+
+This module centralizes best-effort debug logging used by download post-processing
+and atomic filesystem operations.
+
+It is intentionally defensive: failures collecting context should never mask the
+original error.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def _format_uid(uid: int) -> str:
+    try:
+        import pwd
+
+        return pwd.getpwuid(uid).pw_name
+    except Exception:
+        return str(uid)
+
+
+def _format_gid(gid: int) -> str:
+    try:
+        import grp
+
+        return grp.getgrgid(gid).gr_name
+    except Exception:
+        return str(gid)
+
+
+def log_path_permission_context(label: str, path: Path) -> None:
+    """Log useful permission/ownership context for a path.
+
+    Only call this from failure paths.
+    """
+
+    try:
+        euid = os.geteuid() if hasattr(os, "geteuid") else None
+        egid = os.getegid() if hasattr(os, "getegid") else None
+        groups = os.getgroups() if hasattr(os, "getgroups") else []
+
+        if euid is not None and egid is not None:
+            logger.debug(
+                "Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s",
+                label,
+                _format_uid(euid),
+                euid,
+                _format_gid(egid),
+                egid,
+                [f"{_format_gid(g)}({g})" for g in groups],
+            )
+
+        for probe in [path, path.parent]:
+            try:
+                resolved = probe.resolve()
+            except Exception:
+                resolved = probe
+
+            try:
+                st = probe.stat()
+                logger.debug(
+                    "Path permissions (%s): path=%s resolved=%s mode=%s owner=%s(%d) group=%s(%d) dir=%s symlink=%s",
+                    label,
+                    probe,
+                    resolved,
+                    oct(st.st_mode & 0o777),
+                    _format_uid(st.st_uid),
+                    st.st_uid,
+                    _format_gid(st.st_gid),
+                    st.st_gid,
+                    probe.is_dir(),
+                    probe.is_symlink(),
+                )
+            except Exception as stat_error:
+                logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
+    except Exception as context_error:
+        logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
+
+
+def log_transfer_permission_context(label: str, source: Path, dest: Path, error: Exception) -> None:
+    """Log useful permission/ownership context when a file transfer fails."""
+
+    try:
+        euid = os.geteuid() if hasattr(os, "geteuid") else None
+        egid = os.getegid() if hasattr(os, "getegid") else None
+        groups = os.getgroups() if hasattr(os, "getgroups") else []
+
+        if euid is not None and egid is not None:
+            logger.debug(
+                "Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s error=%s",
+                label,
+                _format_uid(euid),
+                euid,
+                _format_gid(egid),
+                egid,
+                [f"{_format_gid(g)}({g})" for g in groups],
+                error,
+            )
+
+        for probe in [source, dest, dest.parent]:
+            try:
+                st = probe.stat()
+                logger.debug(
+                    "Path permissions (%s): path=%s mode=%s owner=%s(%d) group=%s(%d) exists=%s dir=%s",
+                    label,
+                    probe,
+                    oct(st.st_mode & 0o777),
+                    _format_uid(st.st_uid),
+                    st.st_uid,
+                    _format_gid(st.st_gid),
+                    st.st_gid,
+                    probe.exists(),
+                    probe.is_dir(),
+                )
+            except Exception as stat_error:
+                logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
+    except Exception as context_error:
+        logger.debug("Permission context (%s): failed to collect: %s", label, context_error)

shelfmark/download/postprocess/__init__.py

@@ -0,0 +1,11 @@
+"""Post-download processing pipeline.
+
+This package contains the post-download processing pipeline (staging, scanning,
+archive extraction, transfers, and safe cleanup) and the router that selects an
+output handler.
+
+Output handlers live in `shelfmark.download.outputs` and should depend on
+`pipeline` (not `router`) to avoid circular imports.
+"""
+
+from .router import post_process_download

shelfmark/download/postprocess/destination.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import (
+    get_aa_content_type_dir,
+    get_destination,
+    is_audiobook as check_audiobook,
+)
+from shelfmark.download.permissions_debug import log_path_permission_context
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def validate_destination(destination: Path, status_callback) -> bool:
+    """Validate destination path is absolute, exists, and writable."""
+
+    if not destination.is_absolute():
+        logger.warning(f"Destination must be absolute: {destination}")
+        status_callback("error", f"Destination must be absolute: {destination}")
+        return False
+
+    if destination.exists() and not destination.is_dir():
+        logger.warning(f"Destination is not a directory: {destination}")
+        status_callback("error", f"Destination is not a directory: {destination}")
+        return False
+
+    if not destination.exists():
+        try:
+            destination.mkdir(parents=True, exist_ok=True)
+        except (OSError, PermissionError) as exc:
+            log_path_permission_context("destination_create", destination)
+            logger.warning(f"Cannot create destination: {destination} ({exc})")
+            status_callback("error", f"Cannot create destination: {destination} ({exc})")
+            return False
+
+    test_path = destination / f".shelfmark_write_test_{uuid.uuid4().hex}.tmp"
+
+    try:
+        test_content = (
+            f"This file was created to verify if '{destination}' is writable. "
+            "It should've been automatically deleted. Feel free to delete it.\n"
+        )
+        test_path.write_text(test_content)
+        test_path.unlink(missing_ok=True)
+    except Exception as exc:
+        logger.debug("Destination write probe path: %s", test_path)
+        log_path_permission_context("destination_write_probe", destination)
+        logger.warning(f"Destination not writable: {destination} ({exc})")
+        status_callback("error", f"Destination not writable: {destination} ({exc})")
+        return False
+
+    return True
+
+
+def get_final_destination(task: DownloadTask) -> Path:
+    """Get final destination directory, with content-type routing support."""
+
+    is_audiobook = check_audiobook(task.content_type)
+
+    if task.source == "direct_download" and not is_audiobook:
+        override = get_aa_content_type_dir(task.content_type)
+        if override:
+            return override
+
+    return get_destination(is_audiobook)

shelfmark/download/postprocess/pipeline.py

@@ -0,0 +1,76 @@
+"""Post-download processing pipeline.
+
+This module is the public API surface for post-download processing.
+
+Implementation lives in submodules in this package:
+
+- `types`: dataclasses used across the pipeline
+- `workspace`: managed workspace + cleanup rules
+- `scan`: directory scanning + archive extraction
+- `transfer`: hardlink/copy/move + naming/organization
+- `prepare`: staging plan + prepared file selection
+- `steps`: lightweight plan logging helpers
+
+Keeping this file as a facade avoids churn in call sites while letting the
+implementation stay modular.
+"""
+
+from __future__ import annotations
+
+from .destination import get_final_destination, validate_destination
+from .prepare import build_output_plan, prepare_output_files
+from .scan import (
+    collect_directory_files,
+    collect_staged_files,
+    extract_archive_files,
+    get_supported_formats,
+    scan_directory_tree,
+)
+from .steps import log_plan_steps, record_step
+from .transfer import (
+    build_metadata_dict,
+    is_torrent_source,
+    process_directory,
+    resolve_hardlink_source,
+    should_hardlink,
+    transfer_book_files,
+    transfer_directory_to_library,
+    transfer_file_to_library,
+)
+from .types import OutputPlan, PlanStep, PreparedFiles, TransferPlan
+from .workspace import (
+    cleanup_output_staging,
+    is_managed_workspace_path,
+    is_within_tmp_dir,
+    safe_cleanup_path,
+)
+
+__all__ = [
+    "OutputPlan",
+    "PlanStep",
+    "PreparedFiles",
+    "TransferPlan",
+    "build_metadata_dict",
+    "build_output_plan",
+    "cleanup_output_staging",
+    "collect_directory_files",
+    "collect_staged_files",
+    "extract_archive_files",
+    "get_final_destination",
+    "get_supported_formats",
+    "is_managed_workspace_path",
+    "is_torrent_source",
+    "is_within_tmp_dir",
+    "log_plan_steps",
+    "prepare_output_files",
+    "process_directory",
+    "record_step",
+    "resolve_hardlink_source",
+    "safe_cleanup_path",
+    "scan_directory_tree",
+    "should_hardlink",
+    "transfer_book_files",
+    "transfer_directory_to_library",
+    "transfer_file_to_library",
+    "validate_destination",
+]

shelfmark/download/postprocess/policy.py

@@ -0,0 +1,100 @@
+"""Post-download processing policy.
+
+This module holds configuration-driven *policy* decisions that are shared across
+post-download processing components, but are not specific to archive extraction.
+
+Examples:
+- Which file formats are enabled
+- How files should be organized (none/rename/organize)
+- Which naming templates to use
+
+Implementation note:
+Keep this module free of dependencies on archive extraction mechanics to avoid
+circular imports (`archive` is used by the pipeline).
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+import shelfmark.core.config as core_config
+
+
+def get_supported_formats() -> List[str]:
+    """Get current supported formats from config singleton."""
+
+    formats = core_config.config.get(
+        "SUPPORTED_FORMATS",
+        ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
+    )
+
+    # Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
+    if isinstance(formats, str):
+        return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
+
+    return [fmt.lower() for fmt in formats]
+
+
+def get_supported_audiobook_formats() -> List[str]:
+    """Get current supported audiobook formats from config singleton."""
+
+    formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
+
+    # Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
+    if isinstance(formats, str):
+        return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
+
+    return [fmt.lower() for fmt in formats]
+
+
+def get_file_organization(is_audiobook: bool) -> str:
+    """Get the file organization mode for the content type."""
+
+    key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
+    mode = core_config.config.get(key, "rename")
+
+    # Handle legacy settings migration
+    if mode not in ("none", "rename", "organize"):
+        legacy_key = "PROCESSING_MODE_AUDIOBOOK" if is_audiobook else "PROCESSING_MODE"
+        legacy_mode = core_config.config.get(legacy_key, "ingest")
+        if legacy_mode == "library":
+            return "organize"
+        if core_config.config.get("USE_BOOK_TITLE", True):
+            return "rename"
+        return "none"
+
+    return mode
+
+
+def get_template(is_audiobook: bool, organization_mode: str) -> str:
+    """Get the template for the content type and organization mode."""
+
+    # Determine the correct key based on content type and organization mode
+    if is_audiobook:
+        if organization_mode == "organize":
+            key = "TEMPLATE_AUDIOBOOK_ORGANIZE"
+        else:
+            key = "TEMPLATE_AUDIOBOOK_RENAME"
+    else:
+        if organization_mode == "organize":
+            key = "TEMPLATE_ORGANIZE"
+        else:
+            key = "TEMPLATE_RENAME"
+
+    template = core_config.config.get(key, "")
+
+    # Fallback to legacy keys if new keys are empty
+    if not template:
+        legacy_key = "TEMPLATE_AUDIOBOOK" if is_audiobook else "TEMPLATE"
+        template = core_config.config.get(legacy_key, "")
+
+    if not template:
+        legacy_key = "LIBRARY_TEMPLATE_AUDIOBOOK" if is_audiobook else "LIBRARY_TEMPLATE"
+        template = core_config.config.get(legacy_key, "")
+
+    if not template:
+        if organization_mode == "organize":
+            return "{Author}/{Title} ({Year})"
+        return "{Author} - {Title} ({Year})"
+
+    return template

shelfmark/download/postprocess/prepare.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.download.archive import is_archive
+from shelfmark.download.staging import STAGE_COPY, STAGE_NONE, get_staging_dir, stage_path
+
+from .scan import collect_staged_files
+from .transfer import resolve_hardlink_source
+from .types import OutputPlan, PreparedFiles
+from .workspace import cleanup_output_staging, is_managed_workspace_path
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def build_output_plan(
+    temp_file: Path,
+    task: DownloadTask,
+    output_mode: str,
+    destination: Optional[Path] = None,
+    status_callback=None,
+) -> OutputPlan:
+    """Build an output plan that describes staging behavior for file-based outputs."""
+
+    transfer_plan = resolve_hardlink_source(temp_file, task, destination, status_callback)
+    runs_custom_script = bool(core_config.config.CUSTOM_SCRIPT) and temp_file.is_file() and not is_archive(temp_file)
+
+    stage_action = STAGE_COPY if runs_custom_script and not is_managed_workspace_path(temp_file) else STAGE_NONE
+    staging_dir = get_staging_dir()
+
+    return OutputPlan(
+        mode=output_mode,
+        stage_action=stage_action,
+        staging_dir=staging_dir,
+        allow_archive_extraction=transfer_plan.allow_archive_extraction,
+        transfer_plan=transfer_plan,
+    )
+
+
+def prepare_output_files(
+    temp_file: Path,
+    task: DownloadTask,
+    output_mode: str,
+    status_callback,
+    destination: Optional[Path] = None,
+    output_plan: Optional[OutputPlan] = None,
+) -> Optional[PreparedFiles]:
+    if output_plan is None:
+        output_plan = build_output_plan(
+            temp_file,
+            task,
+            output_mode=output_mode,
+            destination=destination,
+            status_callback=status_callback,
+        )
+
+    working_path = temp_file
+    if output_plan.stage_action != STAGE_NONE:
+        step_label = "Staging torrent files" if output_plan.stage_action == STAGE_COPY else "Staging files"
+        status_callback("resolving", step_label)
+        working_path = stage_path(working_path, output_plan.staging_dir, output_plan.stage_action)
+
+    can_delete_source_archives = output_plan.stage_action != STAGE_NONE or is_managed_workspace_path(working_path)
+
+    files, rejected_files, cleanup_paths, error = collect_staged_files(
+        working_path=working_path,
+        task=task,
+        allow_archive_extraction=output_plan.allow_archive_extraction,
+        status_callback=status_callback,
+        cleanup_archives=can_delete_source_archives,
+    )
+
+    if error:
+        status_callback("error", error)
+        cleanup_output_staging(output_plan, working_path, task, cleanup_paths)
+        return None
+
+    if output_plan.stage_action == STAGE_NONE and is_managed_workspace_path(working_path):
+        cleanup_paths = [*cleanup_paths, working_path]
+
+    return PreparedFiles(
+        output_plan=output_plan,
+        working_path=working_path,
+        files=files,
+        rejected_files=rejected_files,
+        cleanup_paths=cleanup_paths,
+    )

shelfmark/download/postprocess/router.py

@@ -0,0 +1,52 @@
+"""Output routing for post-download processing.
+
+This module selects the appropriate output handler and invokes it.
+
+Keeping this separate from `pipeline.py` avoids circular imports:
+
+- output handlers depend on `pipeline`
+- router depends on the output registry
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from threading import Event
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask, SearchMode
+from shelfmark.download.outputs import resolve_output_handler
+
+logger = setup_logger(__name__)
+
+
+def post_process_download(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    """Post-process download using the selected output handler."""
+
+    if task.search_mode is None:
+        logger.warning(
+            "Task %s: missing search_mode; defaulting to Direct mode behavior",
+            task.task_id,
+        )
+    elif task.search_mode not in (SearchMode.DIRECT, SearchMode.UNIVERSAL):
+        logger.warning(
+            "Task %s: invalid search_mode=%s; defaulting to Direct mode behavior",
+            task.task_id,
+            task.search_mode,
+        )
+
+    output_handler = resolve_output_handler(task)
+    if output_handler:
+        logger.info("Task %s: using output mode %s", task.task_id, output_handler.mode)
+        return output_handler.handler(temp_file, task, cancel_flag, status_callback)
+
+    from shelfmark.download.outputs.folder import process_folder_output
+
+    logger.info("Task %s: using output mode folder", task.task_id)
+    return process_folder_output(temp_file, task, cancel_flag, status_callback)

shelfmark/download/postprocess/scan.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
+from shelfmark.download.permissions_debug import log_path_permission_context
+from shelfmark.download.postprocess.policy import (
+    get_supported_audiobook_formats,
+    get_supported_formats as get_book_formats,
+)
+from shelfmark.download.staging import build_staging_dir
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def get_supported_formats(content_type: Optional[str] = None) -> List[str]:
+    if check_audiobook(content_type):
+        return get_supported_audiobook_formats()
+    return get_book_formats()
+
+
+def _format_not_supported_error(rejected_files: List[Path], task: DownloadTask) -> str:
+    content_type = task.content_type
+    file_type_label = "audiobook" if check_audiobook(content_type) else "book"
+    rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+    rejected_list = ", ".join(rejected_exts)
+    supported_formats = get_supported_formats(content_type)
+
+    logger.warning(
+        "Task %s: found %d %s(s) but none match supported formats. Rejected formats: %s. Supported: %s",
+        task.task_id,
+        len(rejected_files),
+        file_type_label,
+        rejected_list,
+        ", ".join(sorted(supported_formats)),
+    )
+
+    return (
+        f"Found {len(rejected_files)} {file_type_label}(s) but format not supported ({rejected_list}). "
+        "Enable in Settings > Formats."
+    )
+
+
+def extract_archive_files(
+    archive_path: Path,
+    output_dir: Path,
+    task: DownloadTask,
+    cleanup_archive: bool,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    content_type = task.content_type
+
+    try:
+        extracted_files, warnings, rejected_files = extract_archive(archive_path, output_dir, content_type)
+    except ArchiveExtractionError as exc:
+        logger.warning(
+            "Task %s: archive extraction failed for %s: %s",
+            task.task_id,
+            archive_path.name,
+            exc,
+        )
+        return [], [], [], str(exc)
+
+    if warnings:
+        logger.debug(
+            "Task %s: archive warnings for %s: %s",
+            task.task_id,
+            archive_path.name,
+            "; ".join(warnings),
+        )
+
+    if cleanup_archive:
+        archive_path.unlink(missing_ok=True)
+
+    cleanup_paths = [output_dir]
+
+    if not extracted_files:
+        if rejected_files:
+            return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
+        file_type_label = "audiobook" if check_audiobook(content_type) else "book"
+        return [], rejected_files, cleanup_paths, f"No {file_type_label} files found in archive"
+
+    logger.debug(
+        "Task %s: extracted %d file(s) from archive %s",
+        task.task_id,
+        len(extracted_files),
+        archive_path.name,
+    )
+
+    return extracted_files, rejected_files, cleanup_paths, None
+
+
+def scan_directory_tree(
+    directory: Path,
+    content_type: Optional[str],
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    """Scan a directory tree for book files, trackable-but-unsupported files, and archives."""
+
+    try:
+        with os.scandir(directory) as it:
+            next(it, None)
+    except PermissionError as exc:
+        log_path_permission_context("scan_directory", directory)
+        logger.warning(f"Permission denied scanning directory: {directory} ({exc})")
+        return [], [], [], f"Permission denied accessing download folder: {directory}"
+    except (FileNotFoundError, NotADirectoryError, OSError) as exc:
+        logger.warning(f"Cannot access download folder: {directory} ({exc})")
+        return [], [], [], f"Cannot access download folder: {directory} ({exc})"
+
+    book_files: List[Path] = []
+    rejected_files: List[Path] = []
+    archive_files: List[Path] = []
+
+    supported_formats = get_supported_formats(content_type)
+    supported_exts = {f".{fmt}" for fmt in supported_formats}
+
+    is_audiobook = check_audiobook(content_type)
+    if is_audiobook:
+        trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
+    else:
+        trackable_exts = {
+            '.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
+            '.doc', '.docx', '.rtf', '.txt',
+        }
+
+    logged_walk_permission_context = False
+
+    def onerror(error: OSError) -> None:
+        nonlocal logged_walk_permission_context
+
+        if isinstance(error, PermissionError):
+            if not logged_walk_permission_context:
+                try:
+                    error_path = Path(getattr(error, "filename", "") or str(directory))
+                except Exception:
+                    error_path = directory
+
+                log_path_permission_context("scan_directory_walk", error_path)
+                logged_walk_permission_context = True
+
+            logger.debug(f"Skipping inaccessible path during scan: {error}")
+        else:
+            logger.debug(f"Error scanning directory tree: {error}")
+
+    for root, _, files in os.walk(directory, onerror=onerror):
+        for filename in files:
+            file_path = Path(root) / filename
+            suffix = file_path.suffix.lower()
+
+            if suffix in supported_exts:
+                book_files.append(file_path)
+            elif suffix in trackable_exts:
+                rejected_files.append(file_path)
+
+            if is_archive(file_path):
+                archive_files.append(file_path)
+
+    return book_files, rejected_files, archive_files, None
+
+
+def collect_directory_files(
+    directory: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool,
+    status_callback=None,
+    cleanup_archives: bool = False,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    content_type = task.content_type
+    book_files, rejected_files, archive_files, scan_error = scan_directory_tree(directory, content_type)
+    if scan_error:
+        return [], [], [], scan_error
+
+    if book_files:
+        if archive_files:
+            logger.debug(
+                "Task %s: ignoring %d archive(s) - already have %d book file(s)",
+                task.task_id,
+                len(archive_files),
+                len(book_files),
+            )
+        if rejected_files:
+            rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+            logger.debug(
+                "Task %s: also found %d file(s) with unsupported formats: %s",
+                task.task_id,
+                len(rejected_files),
+                ", ".join(rejected_exts),
+            )
+        return book_files, rejected_files, [], None
+
+    if archive_files:
+        if not allow_archive_extraction:
+            logger.warning(
+                "Task %s: archive extraction disabled (torrent hardlinking enabled) for %s",
+                task.task_id,
+                directory,
+            )
+            return [], rejected_files, [], "Archive extraction is disabled when torrent hardlinking is enabled"
+
+        if status_callback:
+            status_callback("resolving", "Extracting archives")
+
+        logger.info("Task %s: extracting %d archive(s)", task.task_id, len(archive_files))
+
+        all_files: List[Path] = []
+        all_errors: List[str] = []
+        cleanup_paths: List[Path] = []
+
+        for archive in archive_files:
+            extract_dir = build_staging_dir("extract", task.task_id)
+            extracted_files, archive_rejected, archive_cleanup, error = extract_archive_files(
+                archive_path=archive,
+                output_dir=extract_dir,
+                task=task,
+                cleanup_archive=cleanup_archives,
+            )
+
+            if error:
+                all_errors.append(f"{archive.name}: {error}")
+            if archive_rejected:
+                rejected_files.extend(archive_rejected)
+            if extracted_files:
+                all_files.extend(extracted_files)
+            if archive_cleanup:
+                cleanup_paths.extend(archive_cleanup)
+
+        if all_files:
+            logger.info(
+                "Task %s: extracted %d file(s) from %d archive(s)",
+                task.task_id,
+                len(all_files),
+                len(archive_files),
+            )
+            return all_files, rejected_files, cleanup_paths, None
+
+        if all_errors:
+            return [], rejected_files, cleanup_paths, "; ".join(all_errors)
+
+        if rejected_files:
+            return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
+
+        return [], rejected_files, cleanup_paths, "No book files found in archives"
+
+    if rejected_files:
+        return [], rejected_files, [], _format_not_supported_error(rejected_files, task)
+
+    return [], rejected_files, [], "No book files found in download"
+
+
+def collect_staged_files(
+    working_path: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool,
+    status_callback,
+    cleanup_archives: bool,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    if working_path.is_dir():
+        if status_callback:
+            status_callback("resolving", "Processing download folder")
+        return collect_directory_files(
+            working_path,
+            task,
+            allow_archive_extraction=allow_archive_extraction,
+            status_callback=status_callback,
+            cleanup_archives=cleanup_archives,
+        )
+
+    if is_archive(working_path) and allow_archive_extraction:
+        if status_callback:
+            status_callback("resolving", "Extracting archive")
+
+        logger.info("Task %s: extracting archive %s", task.task_id, working_path.name)
+
+        extract_dir = build_staging_dir("extract", task.task_id)
+        extracted_files, rejected_files, cleanup_paths, error = extract_archive_files(
+            archive_path=working_path,
+            output_dir=extract_dir,
+            task=task,
+            cleanup_archive=cleanup_archives,
+        )
+
+        if extracted_files:
+            logger.info(
+                "Task %s: extracted %d file(s) from archive %s",
+                task.task_id,
+                len(extracted_files),
+                working_path.name,
+            )
+
+        return extracted_files, rejected_files, cleanup_paths, error
+
+    # Single-file download result (non-archive).
+    # Ensure we respect the user's supported format settings.
+    suffix = working_path.suffix.lower()
+    supported_formats = get_supported_formats(task.content_type)
+    supported_exts = {f".{fmt}" for fmt in supported_formats}
+
+    is_audiobook = check_audiobook(task.content_type)
+    if is_audiobook:
+        trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
+    else:
+        trackable_exts = {
+            '.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
+            '.doc', '.docx', '.rtf', '.txt',
+        }
+
+    if suffix in supported_exts:
+        return [working_path], [], [], None
+
+    if suffix in trackable_exts:
+        return [], [working_path], [], _format_not_supported_error([working_path], task)
+
+    file_type_label = "audiobook" if is_audiobook else "book"
+    return [], [], [], f"Unsupported {file_type_label} file type: {suffix or working_path.name}"

shelfmark/download/postprocess/steps.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import Any, List
+
+from shelfmark.core.logger import setup_logger
+
+from .types import PlanStep
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def record_step(steps: List[PlanStep], name: str, **details: Any) -> None:
+    steps.append(PlanStep(name=name, details=details))
+
+
+def log_plan_steps(task_id: str, steps: List[PlanStep]) -> None:
+    if not steps:
+        return
+    summary = " -> ".join(step.name for step in steps)
+    logger.debug("Processing plan for %s: %s", task_id, summary)

shelfmark/download/postprocess/transfer.py

@@ -0,0 +1,399 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.naming import (
+    assign_part_numbers,
+    build_library_path,
+    parse_naming_template,
+    same_filesystem,
+    sanitize_filename,
+)
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.fs import atomic_copy, atomic_hardlink, atomic_move
+from shelfmark.download.postprocess.policy import get_file_organization, get_template
+
+from .scan import collect_directory_files, scan_directory_tree
+from .types import TransferPlan
+from .workspace import safe_cleanup_path
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def should_hardlink(task: DownloadTask) -> bool:
+    """Check if hardlinking is enabled for this task (Prowlarr torrents only)."""
+
+    if task.source != "prowlarr":
+        return False
+
+    if not task.original_download_path:
+        return False
+
+    is_audiobook = check_audiobook(task.content_type)
+    key = "HARDLINK_TORRENTS_AUDIOBOOK" if is_audiobook else "HARDLINK_TORRENTS"
+
+    hardlink_enabled = core_config.config.get(key)
+    if hardlink_enabled is None:
+        hardlink_enabled = core_config.config.get("TORRENT_HARDLINK", False)
+
+    return bool(hardlink_enabled)
+
+
+
+def build_metadata_dict(task: DownloadTask) -> dict:
+    return {
+        "Author": task.author,
+        "Title": task.title,
+        "Subtitle": task.subtitle,
+        "Year": task.year,
+        "Series": task.series_name,
+        "SeriesPosition": task.series_position,
+    }
+
+
+def resolve_hardlink_source(
+    temp_file: Path,
+    task: DownloadTask,
+    destination: Optional[Path],
+    status_callback=None,
+) -> TransferPlan:
+    """Resolve hardlink eligibility and source path for transfers."""
+
+    use_hardlink = False
+    source_path = temp_file
+    hardlink_enabled = should_hardlink(task)
+
+    if hardlink_enabled and task.original_download_path:
+        hardlink_source = Path(task.original_download_path)
+        if destination and hardlink_source.exists() and same_filesystem(hardlink_source, destination):
+            use_hardlink = True
+            source_path = hardlink_source
+        elif hardlink_source.exists():
+            logger.warning(
+                f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
+                "Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
+            )
+            if status_callback:
+                status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
+
+    return TransferPlan(
+        source_path=source_path,
+        use_hardlink=use_hardlink,
+        allow_archive_extraction=not hardlink_enabled,
+        hardlink_enabled=hardlink_enabled,
+    )
+
+
+def is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
+    """Check if source is the torrent client path (needs copy to preserve seeding)."""
+
+    if not task.original_download_path:
+        return False
+
+    original_path = Path(task.original_download_path)
+    try:
+        return source_path.resolve() == original_path.resolve()
+    except (OSError, ValueError):
+        try:
+            return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path))
+        except Exception:
+            return False
+
+
+def _max_attempts_for_batch(file_count: int, default: int = 100) -> int:
+    if file_count <= 1:
+        return default
+    return max(default, file_count + default)
+
+
+def _transfer_single_file(
+    source_path: Path,
+    dest_path: Path,
+    use_hardlink: bool,
+    is_torrent: bool,
+    preserve_source: bool = False,
+    max_attempts: int = 100,
+) -> Tuple[Path, str]:
+    if use_hardlink:
+        final_path = atomic_hardlink(source_path, dest_path, max_attempts=max_attempts)
+        try:
+            if os.stat(source_path).st_ino == os.stat(final_path).st_ino:
+                return final_path, "hardlink"
+        except OSError:
+            return final_path, "hardlink"
+        return final_path, "copy"
+
+    if is_torrent or preserve_source:
+        return atomic_copy(source_path, dest_path, max_attempts=max_attempts), "copy"
+
+    return atomic_move(source_path, dest_path, max_attempts=max_attempts), "move"
+
+
+def transfer_book_files(
+    book_files: List[Path],
+    destination: Path,
+    task: DownloadTask,
+    use_hardlink: bool,
+    is_torrent: bool,
+    preserve_source: bool = False,
+    organization_mode: Optional[str] = None,
+) -> Tuple[List[Path], Optional[str]]:
+    if not book_files:
+        return [], "No book files found"
+
+    is_audiobook = check_audiobook(task.content_type)
+    organization_mode = organization_mode or get_file_organization(is_audiobook)
+    max_attempts = _max_attempts_for_batch(len(book_files))
+
+    final_paths: List[Path] = []
+
+    if organization_mode == "organize":
+        template = get_template(is_audiobook, "organize")
+        metadata = build_metadata_dict(task)
+
+        if len(book_files) == 1:
+            source_file = book_files[0]
+            ext = source_file.suffix.lstrip(".") or task.format or ""
+            dest_path = build_library_path(str(destination), template, metadata, extension=ext or None)
+            dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+            final_path, op = _transfer_single_file(
+                source_file,
+                dest_path,
+                use_hardlink,
+                is_torrent,
+                preserve_source=preserve_source,
+                max_attempts=max_attempts,
+            )
+            final_paths.append(final_path)
+            logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
+        else:
+            zero_pad_width = max(len(str(len(book_files))), 2)
+            files_with_parts = assign_part_numbers(book_files, zero_pad_width)
+
+            for source_file, part_number in files_with_parts:
+                ext = source_file.suffix.lstrip(".") or task.format or ""
+                file_metadata = {**metadata, "PartNumber": part_number}
+                dest_path = build_library_path(str(destination), template, file_metadata, extension=ext or None)
+                dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+                final_path, op = _transfer_single_file(
+                    source_file,
+                    dest_path,
+                    use_hardlink,
+                    is_torrent,
+                    preserve_source=preserve_source,
+                    max_attempts=max_attempts,
+                )
+                final_paths.append(final_path)
+                logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
+
+        return final_paths, None
+
+    for book_file in book_files:
+        if len(book_files) == 1 and organization_mode != "none":
+            if not task.format:
+                task.format = book_file.suffix.lower().lstrip(".")
+
+            template = get_template(is_audiobook, "rename")
+            metadata = build_metadata_dict(task)
+            extension = book_file.suffix.lstrip(".") or task.format or ""
+
+            filename = parse_naming_template(template, metadata, allow_path_separators=False)
+            filename = Path(filename).name if filename else ""
+            if filename and extension:
+                filename = f"{sanitize_filename(filename)}.{extension}"
+            else:
+                filename = book_file.name
+        else:
+            filename = book_file.name
+
+        dest_path = destination / filename
+        final_path, op = _transfer_single_file(
+            book_file,
+            dest_path,
+            use_hardlink,
+            is_torrent,
+            preserve_source=preserve_source,
+            max_attempts=max_attempts,
+        )
+        final_paths.append(final_path)
+        logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
+
+    return final_paths, None
+
+
+def process_directory(
+    directory: Path,
+    ingest_dir: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool = True,
+    use_hardlink: Optional[bool] = None,
+) -> Tuple[List[Path], Optional[str]]:
+    """Process staged directory: find book files, extract archives, move to ingest."""
+
+    try:
+        is_torrent = is_torrent_source(directory, task)
+        book_files, _, cleanup_paths, error = collect_directory_files(
+            directory,
+            task,
+            allow_archive_extraction=allow_archive_extraction,
+            status_callback=None,
+            cleanup_archives=not is_torrent,
+        )
+
+        if error:
+            if not is_torrent:
+                safe_cleanup_path(directory, task)
+                for cleanup_path in cleanup_paths:
+                    safe_cleanup_path(cleanup_path, task)
+            return [], error
+
+        if use_hardlink is None:
+            use_hardlink = should_hardlink(task)
+
+        final_paths, error = transfer_book_files(
+            book_files,
+            destination=ingest_dir,
+            task=task,
+            use_hardlink=use_hardlink,
+            is_torrent=is_torrent,
+        )
+
+        if error:
+            return [], error
+
+        if not is_torrent:
+            safe_cleanup_path(directory, task)
+            for cleanup_path in cleanup_paths:
+                safe_cleanup_path(cleanup_path, task)
+
+        return final_paths, None
+
+    except Exception as exc:
+        logger.error_trace("Task %s: error processing directory %s: %s", task.task_id, directory, exc)
+        if not is_torrent_source(directory, task):
+            safe_cleanup_path(directory, task)
+        return [], str(exc)
+
+
+def transfer_file_to_library(
+    source_path: Path,
+    library_base: str,
+    template: str,
+    metadata: dict,
+    task: DownloadTask,
+    temp_file: Optional[Path],
+    status_callback,
+    use_hardlink: bool,
+) -> Optional[str]:
+    extension = source_path.suffix.lstrip(".") or task.format
+    dest_path = build_library_path(library_base, template, metadata, extension)
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+    is_torrent = is_torrent_source(source_path, task)
+    final_path, op = _transfer_single_file(
+        source_path,
+        dest_path,
+        use_hardlink,
+        is_torrent,
+        max_attempts=_max_attempts_for_batch(1),
+    )
+    logger.info(f"Library {op}: {final_path}")
+
+    if use_hardlink and temp_file and not is_torrent_source(temp_file, task):
+        safe_cleanup_path(temp_file, task)
+
+    status_callback("complete", "Complete")
+    return str(final_path)
+
+
+def transfer_directory_to_library(
+    source_dir: Path,
+    library_base: str,
+    template: str,
+    metadata: dict,
+    task: DownloadTask,
+    temp_file: Optional[Path],
+    status_callback,
+    use_hardlink: bool,
+) -> Optional[str]:
+    content_type = task.content_type.lower() if task.content_type else None
+    source_files, _, _, scan_error = scan_directory_tree(source_dir, content_type)
+    if scan_error:
+        logger.warning(scan_error)
+        status_callback("error", scan_error)
+        if temp_file:
+            safe_cleanup_path(temp_file, task)
+        return None
+
+    if not source_files:
+        logger.warning(f"No supported files in {source_dir.name}")
+        status_callback("error", "No supported file formats found")
+        if temp_file:
+            safe_cleanup_path(temp_file, task)
+        return None
+
+    base_library_path = build_library_path(library_base, template, metadata, extension=None)
+    base_library_path.parent.mkdir(parents=True, exist_ok=True)
+
+    is_torrent = is_torrent_source(source_dir, task)
+    transferred_paths: List[Path] = []
+    max_attempts = _max_attempts_for_batch(len(source_files))
+
+    if len(source_files) == 1:
+        source_file = source_files[0]
+        ext = source_file.suffix.lstrip(".")
+        dest_path = base_library_path.with_suffix(f".{ext}")
+        final_path, op = _transfer_single_file(
+            source_file,
+            dest_path,
+            use_hardlink,
+            is_torrent,
+            max_attempts=max_attempts,
+        )
+        logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
+        transferred_paths.append(final_path)
+    else:
+        zero_pad_width = max(len(str(len(source_files))), 2)
+        files_with_parts = assign_part_numbers(source_files, zero_pad_width)
+
+        for source_file, part_number in files_with_parts:
+            ext = source_file.suffix.lstrip(".")
+            file_metadata = {**metadata, "PartNumber": part_number}
+            file_path = build_library_path(library_base, template, file_metadata, extension=ext)
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            final_path, op = _transfer_single_file(
+                source_file,
+                file_path,
+                use_hardlink,
+                is_torrent,
+                max_attempts=max_attempts,
+            )
+            logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
+            transferred_paths.append(final_path)
+
+    if use_hardlink:
+        operation = "hardlinks"
+    elif is_torrent:
+        operation = "copies"
+    else:
+        operation = "files"
+    logger.info(f"Created {len(transferred_paths)} library {operation} in {base_library_path.parent}")
+
+    if use_hardlink and temp_file and not is_torrent_source(temp_file, task):
+        safe_cleanup_path(temp_file, task)
+    elif not is_torrent:
+        safe_cleanup_path(temp_file, task)
+        safe_cleanup_path(source_dir, task)
+
+    message = f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
+    status_callback("complete", message)
+
+    return str(transferred_paths[0])