Fix: HTTP grab behavior, logging enforcement (#521)

- Base URL option in settings for reverse proxy setups
- Fix NZB downloads not deleting on completion
- Fix handling for audiobook files over 100+ parts
- Fix prowlarr search timeout 
- Fix prowlarr categorisation for expanded searches

.dockerignore

@@ -37,3 +37,13 @@ dist/
 venv/
 .venv/
 env/
+
+# Frontend build artifacts (built in separate stage)
+src/frontend/node_modules/
+src/frontend/dist/
+src/frontend/.vite/
+
+# Old frontend code (replaced by src/frontend)
+templates/
+static/css/
+static/js/

.github/workflows/build-and-publish-docker-image.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository_owner }}/shelfmark
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -20,12 +20,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - suffix: ""
-            target: cwa-bd
-            image_name_suffix: ""
-          - suffix: "-tor"
-            target: cwa-bd-tor
-            image_name_suffix: "-tor"
+          - target: shelfmark
+          - target: shelfmark-lite
+            image_name_suffix: "-lite"
     steps:
       - name: Get current date
         id: date
@@ -67,6 +64,7 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           build-args: |
             BUILD_VERSION=${{ steps.date.outputs.date }}-${{ github.sha }}
+            RELEASE_VERSION=${{ github.ref_name }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
       
@@ -76,4 +74,71 @@ jobs:
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
+
+  # Create aliases for backwards compatibility
+  create-aliases:
+    needs: build-and-push-images
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    permissions:
+      contents: read
+      packages: write
+    env:
+      # Legacy name for backwards compatibility (hardcoded so it works after rename)
+      LEGACY_NAME: calibre-web-automated-book-downloader
+    steps:
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create legacy aliases
+        run: |
+          # Current image names (follows repo name)
+          STANDARD="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          LITE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-lite"
+
+          # Legacy image names (hardcoded for backwards compatibility)
+          LEGACY="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.LEGACY_NAME }}"
+          LEGACY_TOR="${LEGACY}-tor"
+          LEGACY_EXTBP="${LEGACY}-extbp"
+
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+
+          # Helper function to create alias with all standard tags
+          create_alias() {
+            local SOURCE=$1
+            local ALIAS=$2
+
+            # Always create SHA tag
+            docker buildx imagetools create -t "${ALIAS}:sha-${SHA_SHORT}" "${SOURCE}:sha-${SHA_SHORT}"
+
+            if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+              VERSION="${{ github.ref_name }}"
+              VERSION_NUM="${VERSION#v}"
+              MINOR="${VERSION_NUM%.*}"
+
+              docker buildx imagetools create -t "${ALIAS}:latest" "${SOURCE}:latest"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION_NUM}" "${SOURCE}:${VERSION_NUM}"
+              docker buildx imagetools create -t "${ALIAS}:${MINOR}" "${SOURCE}:${MINOR}"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION}" "${SOURCE}:${VERSION}"
+            else
+              docker buildx imagetools create -t "${ALIAS}:dev" "${SOURCE}:dev"
+            fi
+          }
+
+          # Create legacy aliases pointing to current images
+          # calibre-web-automated-book-downloader → standard image
+          create_alias "${STANDARD}" "${LEGACY}"
+
+          # calibre-web-automated-book-downloader-tor → standard image
+          create_alias "${STANDARD}" "${LEGACY_TOR}"
+
+          # calibre-web-automated-book-downloader-extbp → lite image
+          create_alias "${LITE}" "${LEGACY_EXTBP}"

.gitignore

@@ -227,3 +227,8 @@ pyrightconfig.json
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python
 /downloaded_files
+/.local/
+*.local.*
+AGENTS.md
+.claude/
+.playwright-mcp/

.vscode/launch.json

@@ -2,20 +2,21 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Current CWABD File",
+      "name": "Python Debugger: Current Shelfmark File",
       "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false,
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader",
-        "TEMP_DIR": "/tmp/cwa-book-downloader",
+        "INGEST_DIR": "/tmp/shelfmark",
+        "TEMP_DIR": "/tmp/shelfmark",
         "LOG_LEVEL": "DEBUG",
-        "LOG_ROOT": "/tmp/cwa-book-downloader",
+        "LOG_ROOT": "/tmp/shelfmark",
         "ENABLE_LOGGING": "true",
         "DOCKERMODE": "false",
-        "DEBUG": "true"
+        "DEBUG": "true",
+        "CUSTOM_DNS": "google",
       },
     },
     {
@@ -26,7 +27,7 @@
       "preLaunchTask": "docker-compose up (dev)", // Spin up dev containers
       "postDebugTask": "docker-compose down (dev)", // Optional: tear them down
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -37,7 +38,7 @@
       "preLaunchTask": "docker-compose up (prod)",
       "postDebugTask": "docker-compose down (prod)",
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -53,9 +54,9 @@
   ],
   "compounds": [
     {
-      "name": "Launch CWA-BD",
+      "name": "Launch Shelfmark",
       "configurations": [
-        "Launch cwa-bd app.py",
+        "Launch Shelfmark app.py",
         "Launch Browser"
       ]
     }

Dockerfile

@@ -1,9 +1,37 @@
+ARG TARGETPLATFORM
+ARG TARGETARCH
+ARG BUILDPLATFORM
+ARG BUILDARCH
+
+# Frontend build stage.
+FROM --platform=$BUILDPLATFORM node:20-alpine AS frontend-builder
+
+# Helpful debug output to see what platforms BuildKit thinks it's using
+RUN echo "BUILDPLATFORM=$BUILDPLATFORM BUILDARCH=$BUILDARCH TARGETPLATFORM=$TARGETPLATFORM TARGETARCH=$TARGETARCH"
+
+WORKDIR /frontend
+
+# Copy frontend package files
+COPY src/frontend/package*.json ./
+
+# Install dependencies (cache mount for faster rebuilds)
+RUN --mount=type=cache,target=/root/.npm \
+    npm ci
+
+# Copy frontend source
+COPY src/frontend/ ./
+
+# Build the frontend
+RUN npm run build
+
 # Use python-slim as the base image
 FROM python:3.10-slim AS base
 
 # Add build argument for version
 ARG BUILD_VERSION
 ENV BUILD_VERSION=${BUILD_VERSION}
+ARG RELEASE_VERSION
+ENV RELEASE_VERSION=${RELEASE_VERSION}
 
 # Set shell to bash with pipefail option
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -17,13 +45,12 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    NAME=Calibre-Web-Automated-Book-Downloader \
+    NAME=Shelfmark \
     PYTHONPATH=/app \
-    # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
+    # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
-    LC_ALL=en_US.UTF-8 \
-    APP_ENV=prod
+    LC_ALL=en_US.UTF-8
 
 # Set ARG for build-time expansion (FLASK_PORT), ENV for runtime access
 ENV FLASK_PORT=8084
@@ -38,18 +65,17 @@ RUN apt-get update && \
     curl \
     # For entrypoint
     dumb-init \
-    # For dumb display
-    xvfb \
-    # For screen recording
-    ffmpeg \
     # For debug
     zip iputils-ping \
     # For user switching
     sudo \
-    # --- Chromium Browser ---
-    chromium-driver \
-    # For tkinter (pyautogui)
-    python3-tk && \
+    # --- Tor support (activated via USING_TOR=true) ---
+    tor \
+    supervisor \
+    iptables && \
+    # Configure iptables alternatives for tor.sh compatibility
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
     # Cleanup APT cache *after* all installs in this layer
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
@@ -66,61 +92,73 @@ RUN apt-get update && \
 WORKDIR /app
 
 # Install Python dependencies using pip
-# Upgrade pip first, then copy requirements and install
-# Copying requirements.txt separately leverages build cache
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
-
-# Add this line to grant read/execute permissions to others
-RUN chmod -R o+rx /usr/bin/chromium && \
-    chmod -R o+rx /usr/bin/chromedriver && \
-    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+# Copying requirements files separately leverages build cache
+# Cache mount persists pip cache between builds for faster installs
+COPY requirements-base.txt requirements-shelfmark.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-base.txt
 
 # Copy application code *after* dependencies are installed
 COPY . .
 
+# Copy built frontend from frontend-builder stage
+COPY --from=frontend-builder /frontend/dist /app/frontend-dist
+
 # Final setup: permissions and directories in one layer
 # Only creating directories and setting executable bits.
 # Ownership will be handled by the entrypoint script.
-RUN mkdir -p /var/log/cwa-book-downloader /cwa-book-ingest && \
+RUN mkdir -p /var/log/shelfmark /books && \
     chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
 
 # Expose the application port
 EXPOSE ${FLASK_PORT}
 
 # Add healthcheck for container status
-# This will run as root initially, but check localhost which should work if the app binds correctly.
+# Uses /api/health which doesn't require authentication
 HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
-    CMD curl -s http://localhost:${FLASK_PORT}/request/api/status > /dev/null || exit 1
+    CMD curl -s http://localhost:${FLASK_PORT}/api/health > /dev/null || exit 1
 
 # Use dumb-init as the entrypoint to handle signals properly
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 
-FROM base AS cwa-bd
+FROM base AS shelfmark
 
-# Default command to run the application entrypoint script
-CMD ["/app/entrypoint.sh"]
-
-FROM base AS cwa-bd-tor
-
-ENV USING_TOR=true
-
-# Install Tor and dependencies
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    # --- Tor ---
-    tor \
-    # --- iptables ---
-    iptables && \
-    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
-    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
-    # Cleanup APT cache *after* all installs in this layer
+    # For dumb display
+    xvfb \
+    # For screen recording
+    ffmpeg \
+    # --- Chromium ---
+    chromium \
+    # --- ChromeDriver ---
+    chromium-driver \
+    # For tkinter (pyautogui)
+    python3-tk \
+    # For RAR extraction
+    unrar-free && \
+    # Create symlink so rarfile library can find unrar
+    ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
+    # Cleanup APT cache
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Override the default command to run Tor
+# Install additional dependencies (requirements file already copied in base stage)
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-shelfmark.txt
+
+# Grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium && \
+    chmod -R o+rx /usr/bin/chromedriver && \
+    chmod -R o+rwx /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+
+# Default command to run the application entrypoint script
+CMD ["/app/entrypoint.sh"]
+
+FROM base AS shelfmark-lite
+
+ENV USING_EXTERNAL_BYPASSER=true
+
 CMD ["/app/entrypoint.sh"]

Makefile

@@ -0,0 +1,84 @@
+.PHONY: help install dev build preview typecheck clean up down docker-build refresh restart
+
+# Frontend directory
+FRONTEND_DIR := src/frontend
+
+# Docker compose file
+COMPOSE_FILE := docker-compose.dev.yml
+
+# Default target
+help:
+	@echo "Available targets:"
+	@echo ""
+	@echo "Frontend:"
+	@echo "  install    - Install frontend dependencies"
+	@echo "  dev        - Start development server"
+	@echo "  build      - Build frontend for production"
+	@echo "  preview    - Preview production build"
+	@echo "  typecheck  - Run TypeScript type checking"
+	@echo "  clean      - Remove node_modules and build artifacts"
+	@echo ""
+	@echo "Backend (Docker):"
+	@echo "  up         - Start backend services"
+	@echo "  down       - Stop backend services"
+	@echo "  restart    - Restart backend services (no rebuild)"
+	@echo "  docker-build - Build Docker image"
+	@echo "  refresh    - Rebuild and restart backend services"
+
+# Install dependencies
+install:
+	@echo "Installing frontend dependencies..."
+	cd $(FRONTEND_DIR) && npm install
+
+# Start development server
+dev:
+	@echo "Starting development server..."
+	cd $(FRONTEND_DIR) && npm run dev
+
+# Build for production
+build:
+	@echo "Building frontend for production..."
+	cd $(FRONTEND_DIR) && npm run build
+
+# Preview production build
+preview:
+	@echo "Previewing production build..."
+	cd $(FRONTEND_DIR) && npm run preview
+
+# Type checking
+typecheck:
+	@echo "Running TypeScript type checking..."
+	cd $(FRONTEND_DIR) && npm run typecheck
+
+# Clean build artifacts and dependencies
+clean:
+	@echo "Cleaning build artifacts and dependencies..."
+	rm -rf $(FRONTEND_DIR)/node_modules
+	rm -rf $(FRONTEND_DIR)/dist
+
+# Start backend services
+up:
+	@echo "Starting backend services..."
+	docker compose -f $(COMPOSE_FILE) up -d
+
+# Stop backend services
+down:
+	@echo "Stopping backend services..."
+	docker compose -f $(COMPOSE_FILE) down
+
+# Build Docker image
+docker-build:
+	@echo "Building Docker image..."
+	docker compose -f $(COMPOSE_FILE) build
+
+# Restart backend services (no rebuild)
+restart:
+	@echo "Restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) restart
+
+# Rebuild and restart backend services
+refresh:
+	@echo "Rebuilding and restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) down
+	docker compose -f $(COMPOSE_FILE) build
+	docker compose -f $(COMPOSE_FILE) up -d

app.py

@@ -1,529 +0,0 @@
-"""Flask web application for book download service with URL rewrite support."""
-
-import logging
-import io, re, os
-import sqlite3
-from functools import wraps
-from flask import Flask, request, jsonify, render_template, send_file, send_from_directory
-from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.security import check_password_hash
-from werkzeug.wrappers import Response
-from flask import url_for as flask_url_for
-import typing
-
-from logger import setup_logger
-from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE
-from env import FLASK_HOST, FLASK_PORT, APP_ENV, CWA_DB_PATH, DEBUG
-import backend
-
-from models import SearchFilters
-
-logger = setup_logger(__name__)
-app = Flask(__name__)
-app.wsgi_app = ProxyFix(app.wsgi_app)  # type: ignore
-app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0  # Disable caching
-app.config['APPLICATION_ROOT'] = '/'
-
-# Flask logger
-app.logger.handlers = logger.handlers
-app.logger.setLevel(logger.level)
-# Also handle Werkzeug's logger
-werkzeug_logger = logging.getLogger('werkzeug')
-werkzeug_logger.handlers = logger.handlers
-werkzeug_logger.setLevel(logger.level)
-
-# Set up authentication defaults
-# The secret key will reset every time we restart, which will
-# require users to authenticate again
-app.config.update(
-    SECRET_KEY = os.urandom(64)
-)
-
-def login_required(f):
-    @wraps(f)
-    def decorated_function(*args, **kwargs):
-        # If the CWA_DB_PATH variable exists, but isn't a valid
-        # path, return a server error
-        if CWA_DB_PATH is not None and not os.path.isfile(CWA_DB_PATH):
-            logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path")
-            return Response("Internal Server Error", 500)
-        if not authenticate():
-            return Response(
-                response="Unauthorized",
-                status=401,
-                headers={
-                    "WWW-Authenticate": 'Basic realm="Calibre-Web-Automated-Book-Downloader"',
-                },
-            )
-        return f(*args, **kwargs)
-    return decorated_function
-
-def register_dual_routes(app : Flask) -> None:
-    """
-    Register each route both with and without the /request prefix.
-    This function should be called after all routes are defined.
-    """
-    # Store original url_map rules
-    rules = list(app.url_map.iter_rules())
-    
-    # Add /request prefix to each rule
-    for rule in rules:
-        if rule.rule != '/request/' and rule.rule != '/request':  # Skip if it's already a request route
-            # Create new routes with /request prefix, both with and without trailing slash
-            base_rule = rule.rule[:-1] if rule.rule.endswith('/') else rule.rule
-            if base_rule == '':  # Special case for root path
-                app.add_url_rule('/request', f"root_request", 
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-                app.add_url_rule('/request/', f"root_request_slash", 
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-            else:
-                app.add_url_rule(f"/request{base_rule}", 
-                               f"{rule.endpoint}_request",
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-                app.add_url_rule(f"/request{base_rule}/", 
-                               f"{rule.endpoint}_request_slash",
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-    app.jinja_env.globals['url_for'] = url_for_with_request
-
-def url_for_with_request(endpoint : str, **values : typing.Any) -> str:
-    """Generate URLs with /request prefix by default."""
-    if endpoint == 'static':
-        # For static files, add /request prefix
-        url = flask_url_for(endpoint, **values)
-        return f"/request{url}"
-    return flask_url_for(endpoint, **values)
-
-@app.route('/')
-@login_required
-def index() -> str:
-    """
-    Render main page with search and status table.
-    """
-    return render_template('index.html', book_languages=_SUPPORTED_BOOK_LANGUAGE, default_language=BOOK_LANGUAGE, debug=DEBUG)
-
-@app.route('/favico<path:_>')
-@app.route('/request/favico<path:_>')
-@app.route('/request/static/favico<path:_>')
-def favicon(_ : typing.Any) -> Response:
-    return send_from_directory(os.path.join(app.root_path, 'static', 'media'),
-        'favicon.ico', mimetype='image/vnd.microsoft.icon')
-
-from typing import Union, Tuple
-
-if DEBUG:
-    import subprocess
-    import time
-    from cloudflare_bypasser import _reset_driver as STOP_GUI
-    @app.route('/debug', methods=['GET'])
-    @login_required
-    def debug() -> Union[Response, Tuple[Response, int]]:
-        """
-        This will run the /app/debug.sh script, which will generate a debug zip with all the logs
-        The file will be named /tmp/cwa-book-downloader-debug.zip
-        And then return it to the user
-        """
-        try:
-            # Run the debug script
-            STOP_GUI()
-            time.sleep(1)
-            result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True)
-            if result.returncode != 0:
-                raise Exception(f"Debug script failed: {result.stderr}")
-            logger.info(f"Debug script executed: {result.stdout}")
-            debug_file_path = result.stdout.strip().split('\n')[-1]
-            if not os.path.exists(debug_file_path):
-                logger.error("Debug zip file not found after running debug script")
-                return jsonify({"error": "Failed to generate debug information"}), 500
-                
-            # Return the file to the user
-            return send_file(
-                debug_file_path,
-                mimetype='application/zip',
-                download_name=os.path.basename(debug_file_path),
-                as_attachment=True
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}")
-            return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500
-        except Exception as e:
-            logger.error_trace(f"Debug endpoint error: {e}")
-            return jsonify({"error": str(e)}), 500
-
-if DEBUG:
-    @app.route('/api/restart', methods=['GET'])
-    @login_required
-    def restart() -> Union[Response, Tuple[Response, int]]:
-        """
-        Restart the application
-        """
-        os._exit(0)
-
-@app.route('/api/search', methods=['GET'])
-@login_required
-def api_search() -> Union[Response, Tuple[Response, int]]:
-    """
-    Search for books matching the provided query.
-
-    Query Parameters:
-        query (str): Search term (ISBN, title, author, etc.)
-        isbn (str): Book ISBN
-        author (str): Book Author
-        title (str): Book Title
-        lang (str): Book Language
-        sort (str): Order to sort results
-        content (str): Content type of book
-        format (str): File format filter (pdf, epub, mobi, azw3, fb2, djvu, cbz, cbr)
-
-    Returns:
-        flask.Response: JSON array of matching books or error response.
-    """
-    query = request.args.get('query', '')
-
-    filters = SearchFilters(
-        isbn = request.args.getlist('isbn'),
-        author = request.args.getlist('author'),
-        title = request.args.getlist('title'),
-        lang = request.args.getlist('lang'),
-        sort = request.args.get('sort'),
-        content = request.args.getlist('content'),
-        format = request.args.getlist('format'),
-    )
-
-    if not query and not any(vars(filters).values()):
-        return jsonify([])
-
-    try:
-        books = backend.search_books(query, filters)
-        return jsonify(books)
-    except Exception as e:
-        logger.error_trace(f"Search error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/info', methods=['GET'])
-@login_required
-def api_info() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get detailed book information.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON object with book details, or an error message.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        book = backend.get_book_info(book_id)
-        if book:
-            return jsonify(book)
-        return jsonify({"error": "Book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Info error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download', methods=['GET'])
-@login_required
-def api_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Queue a book for download.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON status object indicating success or failure.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        priority = int(request.args.get('priority', 0))
-        success = backend.queue_book(book_id, priority)
-        if success:
-            return jsonify({"status": "queued", "priority": priority})
-        return jsonify({"error": "Failed to queue book"}), 500
-    except Exception as e:
-        logger.error_trace(f"Download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/status', methods=['GET'])
-@login_required
-def api_status() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current download queue status.
-
-    Returns:
-        flask.Response: JSON object with queue status.
-    """
-    try:
-        status = backend.queue_status()
-        return jsonify(status)
-    except Exception as e:
-        logger.error_trace(f"Status error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/localdownload', methods=['GET'])
-@login_required
-def api_local_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Download an EPUB file from local storage if available.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: The EPUB file if found, otherwise an error response.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        file_data, book_info = backend.get_book_data(book_id)
-        if file_data is None:
-            # Book data not found or not available
-            return jsonify({"error": "File not found"}), 404
-        # Santize the file name
-        file_name = book_info.title
-        file_name = re.sub(r'[\\/:*?"<>|]', '_', file_name.strip())[:245]
-        file_extension = book_info.format
-        # Prepare the file for sending to the client
-        data = io.BytesIO(file_data)
-        return send_file(
-            data,
-            download_name=f"{file_name}.{file_extension}",
-            as_attachment=True
-        )
-
-    except Exception as e:
-        logger.error_trace(f"Local download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download/<book_id>/cancel', methods=['DELETE'])
-@login_required
-def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Cancel a download.
-
-    Path Parameters:
-        book_id (str): Book identifier to cancel
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        success = backend.cancel_download(book_id)
-        if success:
-            return jsonify({"status": "cancelled", "book_id": book_id})
-        return jsonify({"error": "Failed to cancel download or book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Cancel download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/<book_id>/priority', methods=['PUT'])
-@login_required
-def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Set priority for a queued book.
-
-    Path Parameters:
-        book_id (str): Book identifier
-
-    Request Body:
-        priority (int): New priority level (lower number = higher priority)
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'priority' not in data:
-            return jsonify({"error": "Priority not provided"}), 400
-            
-        priority = int(data['priority'])
-        success = backend.set_book_priority(book_id, priority)
-        
-        if success:
-            return jsonify({"status": "updated", "book_id": book_id, "priority": priority})
-        return jsonify({"error": "Failed to update priority or book not found"}), 404
-    except ValueError:
-        return jsonify({"error": "Invalid priority value"}), 400
-    except Exception as e:
-        logger.error_trace(f"Set priority error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/reorder', methods=['POST'])
-@login_required
-def api_reorder_queue() -> Union[Response, Tuple[Response, int]]:
-    """
-    Bulk reorder queue by setting new priorities.
-
-    Request Body:
-        book_priorities (dict): Mapping of book_id to new priority
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'book_priorities' not in data:
-            return jsonify({"error": "book_priorities not provided"}), 400
-            
-        book_priorities = data['book_priorities']
-        if not isinstance(book_priorities, dict):
-            return jsonify({"error": "book_priorities must be a dictionary"}), 400
-            
-        # Validate all priorities are integers
-        for book_id, priority in book_priorities.items():
-            if not isinstance(priority, int):
-                return jsonify({"error": f"Invalid priority for book {book_id}"}), 400
-                
-        success = backend.reorder_queue(book_priorities)
-        
-        if success:
-            return jsonify({"status": "reordered", "updated_count": len(book_priorities)})
-        return jsonify({"error": "Failed to reorder queue"}), 500
-    except Exception as e:
-        logger.error_trace(f"Reorder queue error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/order', methods=['GET'])
-@login_required
-def api_queue_order() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current queue order for display.
-
-    Returns:
-        flask.Response: JSON array of queued books with their order and priorities.
-    """
-    try:
-        queue_order = backend.get_queue_order()
-        return jsonify({"queue": queue_order})
-    except Exception as e:
-        logger.error_trace(f"Queue order error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/downloads/active', methods=['GET'])
-@login_required
-def api_active_downloads() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get list of currently active downloads.
-
-    Returns:
-        flask.Response: JSON array of active download book IDs.
-    """
-    try:
-        active_downloads = backend.get_active_downloads()
-        return jsonify({"active_downloads": active_downloads})
-    except Exception as e:
-        logger.error_trace(f"Active downloads error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/clear', methods=['DELETE'])
-@login_required
-def api_clear_completed() -> Union[Response, Tuple[Response, int]]:
-    """
-    Clear all completed, errored, or cancelled books from tracking.
-
-    Returns:
-        flask.Response: JSON with count of removed books.
-    """
-    try:
-        removed_count = backend.clear_completed()
-        return jsonify({"status": "cleared", "removed_count": removed_count})
-    except Exception as e:
-        logger.error_trace(f"Clear completed error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.errorhandler(404)
-def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 404 (Not Found) errors.
-
-    Args:
-        error (HTTPException): The 404 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 404 status.
-    """
-    logger.warning(f"404 error: {request.url} : {error}")
-    return jsonify({"error": "Resource not found"}), 404
-
-@app.errorhandler(500)
-def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 500 (Internal Server) errors.
-
-    Args:
-        error (HTTPException): The 500 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 500 status.
-    """
-    logger.error_trace(f"500 error: {error}")
-    return jsonify({"error": "Internal server error"}), 500
-
-def authenticate() -> bool:
-    """
-    Helper function that validates Basic credentials
-    against a Calibre-Web app.db SQLite database
-
-    Database structure:
-    - Table 'user' with columns: 'name' (username), 'password'
-    """
-
-    # If the database doesn't exist, the user is always authenticated
-    if not CWA_DB_PATH:
-        return True
-
-    # If no authorization object exists, return false to prompt
-    # a request to the user
-    if not request.authorization:
-        return False
-
-    username = request.authorization.get("username")
-    password = request.authorization.get("password")
-
-    # Validate credentials against database
-    try:
-        # Open database in true read-only mode to avoid journal/WAL writes on RO mounts
-        db_path = os.fspath(CWA_DB_PATH)
-        db_uri = f"file:{db_path}?mode=ro&immutable=1"
-        conn = sqlite3.connect(db_uri, uri=True)
-        cur = conn.cursor()
-        cur.execute("SELECT password FROM user WHERE name = ?", (username,))
-        row = cur.fetchone()
-        conn.close()
-
-        # Check if user exists and password is correct
-        if not row or not row[0] or not check_password_hash(row[0], password):
-            logger.error("User not found or password check failed")
-            return False
-
-    except Exception as e:
-        logger.error_trace(f"CWA DB or authentication send_from_directory: {e}")
-        return False
-
-    logger.info(f"Authentication successful for user {username}")
-    return True
-
-# Register all routes with /request prefix
-register_dual_routes(app)
-
-logger.log_resource_usage()
-
-if __name__ == '__main__':
-    logger.info(f"Starting Flask application on {FLASK_HOST}:{FLASK_PORT} IN {APP_ENV} mode")
-    app.run(
-        host=FLASK_HOST,
-        port=FLASK_PORT,
-        debug=DEBUG 
-    )

backend.py

@@ -1,338 +0,0 @@
-"""Backend logic for the book download application."""
-
-import threading, time
-import shutil
-from pathlib import Path
-from typing import Dict, List, Optional, Any, Tuple
-import subprocess
-import os
-from concurrent.futures import ThreadPoolExecutor, Future
-from threading import Event
-
-from logger import setup_logger
-from config import CUSTOM_SCRIPT
-from env import INGEST_DIR, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE, MAX_CONCURRENT_DOWNLOADS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL
-from models import book_queue, BookInfo, QueueStatus, SearchFilters
-import book_manager
-
-logger = setup_logger(__name__)
-
-def _sanitize_filename(filename: str) -> str:
-    """Sanitize a filename by replacing spaces with underscores and removing invalid characters."""
-    keepcharacters = (' ','.','_')
-    return "".join(c for c in filename if c.isalnum() or c in keepcharacters).rstrip()
-
-def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
-    """Search for books matching the query.
-    
-    Args:
-        query: Search term
-        filters: Search filters object
-        
-    Returns:
-        List[Dict]: List of book information dictionaries
-    """
-    try:
-        books = book_manager.search_books(query, filters)
-        return [_book_info_to_dict(book) for book in books]
-    except Exception as e:
-        logger.error_trace(f"Error searching books: {e}")
-        return []
-
-def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
-    """Get detailed information for a specific book.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Optional[Dict]: Book information dictionary if found
-    """
-    try:
-        book = book_manager.get_book_info(book_id)
-        return _book_info_to_dict(book)
-    except Exception as e:
-        logger.error_trace(f"Error getting book info: {e}")
-        return None
-
-def queue_book(book_id: str, priority: int = 0) -> bool:
-    """Add a book to the download queue with specified priority.
-    
-    Args:
-        book_id: Book identifier
-        priority: Priority level (lower number = higher priority)
-        
-    Returns:
-        bool: True if book was successfully queued
-    """
-    try:
-        book_info = book_manager.get_book_info(book_id)
-        book_queue.add(book_id, book_info, priority)
-        logger.info(f"Book queued with priority {priority}: {book_info.title}")
-        return True
-    except Exception as e:
-        logger.error_trace(f"Error queueing book: {e}")
-        return False
-
-def queue_status() -> Dict[str, Dict[str, Any]]:
-    """Get current status of the download queue.
-    
-    Returns:
-        Dict: Queue status organized by status type
-    """
-    status = book_queue.get_status()
-    # Convert Enum keys to strings and properly format the response
-    return {
-        status_type.value: books
-        for status_type, books in status.items()
-    }
-
-def get_book_data(book_id: str) -> Tuple[Optional[bytes], BookInfo]:
-    """Get book data for a specific book, including its title.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Tuple[Optional[bytes], str]: Book data if available, and the book title
-    """
-    try:
-        book_info = book_queue._book_data[book_id]
-        path = book_info.download_path
-        with open(path, "rb") as f:
-            return f.read(), book_info
-    except Exception as e:
-        logger.error_trace(f"Error getting book data: {e}")
-        if book_info:
-            book_info.download_path = None
-        return None, book_info if book_info else BookInfo(id=book_id, title="Unknown")
-
-def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
-    """Convert BookInfo object to dictionary representation."""
-    return {
-        key: value for key, value in book.__dict__.items()
-        if value is not None
-    }
-
-def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Optional[str]:
-    """Download and process a book with cancellation support.
-    
-    Args:
-        book_id: Book identifier
-        cancel_flag: Threading event to signal cancellation
-        
-    Returns:
-        str: Path to the downloaded book if successful, None otherwise
-    """
-    try:
-        # Check for cancellation before starting
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before starting: {book_id}")
-            return None
-            
-        book_info = book_queue._book_data[book_id]
-        logger.info(f"Starting download: {book_info.title}")
-
-        if USE_BOOK_TITLE:
-            book_name = _sanitize_filename(book_info.title)
-        else:
-            book_name = book_id
-        book_name += f".{book_info.format}"
-        book_path = TMP_DIR / book_name
-
-        # Check cancellation before download
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before book manager call: {book_id}")
-            return None
-
-        # Update progress periodically during download
-        progress_thread = threading.Thread(
-            target=_update_download_progress,
-            args=(book_id, cancel_flag),
-            daemon=True
-        )
-        progress_thread.start()
-
-        success = book_manager.download_book(book_info, book_path)
-        
-        # Stop progress updates
-        cancel_flag.wait(0.1)  # Brief pause for progress thread cleanup
-        
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during download: {book_id}")
-            # Clean up partial download
-            if book_path.exists():
-                book_path.unlink()
-            return None
-            
-        if not success:
-            raise Exception("Unknown error downloading book")
-
-        # Check cancellation before post-processing
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before post-processing: {book_id}")
-            if book_path.exists():
-                book_path.unlink()
-            return None
-
-        if CUSTOM_SCRIPT:
-            logger.info(f"Running custom script: {CUSTOM_SCRIPT}")
-            subprocess.run([CUSTOM_SCRIPT, book_path])
-            
-        intermediate_path = INGEST_DIR / f"{book_id}.crdownload"
-        final_path = INGEST_DIR / book_name
-        
-        if os.path.exists(book_path):
-            logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}")
-            try:
-                shutil.move(book_path, intermediate_path)
-            except Exception as e:
-                logger.debug(f"Error moving book: {e}, will try copying instead")
-                shutil.copy(book_path, intermediate_path)
-                os.remove(book_path)
-            
-            # Final cancellation check before completing
-            if cancel_flag.is_set():
-                logger.info(f"Download cancelled before final rename: {book_id}")
-                if intermediate_path.exists():
-                    intermediate_path.unlink()
-                return None
-                
-            os.rename(intermediate_path, final_path)
-            logger.info(f"Download completed successfully: {book_info.title}")
-            
-        return str(final_path)
-    except Exception as e:
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during error handling: {book_id}")
-        else:
-            logger.error_trace(f"Error downloading book: {e}")
-        return None
-
-def _update_download_progress(book_id: str, cancel_flag: Event) -> None:
-    """Update download progress periodically."""
-    progress = 0.0
-    while not cancel_flag.is_set() and progress < 100.0:
-        # Simulate progress (in real implementation, this would get actual progress)
-        progress = min(100.0, progress + 10.0)
-        book_queue.update_progress(book_id, progress)
-        time.sleep(DOWNLOAD_PROGRESS_UPDATE_INTERVAL)
-
-def cancel_download(book_id: str) -> bool:
-    """Cancel a download.
-    
-    Args:
-        book_id: Book identifier to cancel
-        
-    Returns:
-        bool: True if cancellation was successful
-    """
-    return book_queue.cancel_download(book_id)
-
-def set_book_priority(book_id: str, priority: int) -> bool:
-    """Set priority for a queued book.
-    
-    Args:
-        book_id: Book identifier
-        priority: New priority level (lower = higher priority)
-        
-    Returns:
-        bool: True if priority was successfully changed
-    """
-    return book_queue.set_priority(book_id, priority)
-
-def reorder_queue(book_priorities: Dict[str, int]) -> bool:
-    """Bulk reorder queue.
-    
-    Args:
-        book_priorities: Dict mapping book_id to new priority
-        
-    Returns:
-        bool: True if reordering was successful
-    """
-    return book_queue.reorder_queue(book_priorities)
-
-def get_queue_order() -> List[Dict[str, any]]:
-    """Get current queue order for display."""
-    return book_queue.get_queue_order()
-
-def get_active_downloads() -> List[str]:
-    """Get list of currently active downloads."""
-    return book_queue.get_active_downloads()
-
-def clear_completed() -> int:
-    """Clear all completed downloads from tracking."""
-    return book_queue.clear_completed()
-
-def _process_single_download(book_id: str, cancel_flag: Event) -> None:
-    """Process a single download job."""
-    try:
-        book_queue.update_status(book_id, QueueStatus.DOWNLOADING)
-        download_path = _download_book_with_cancellation(book_id, cancel_flag)
-        
-        if cancel_flag.is_set():
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-            return
-            
-        if download_path:
-            book_queue.update_download_path(book_id, download_path)
-            new_status = QueueStatus.AVAILABLE
-        else:
-            new_status = QueueStatus.ERROR
-            
-        book_queue.update_status(book_id, new_status)
-        
-        logger.info(
-            f"Book {book_id} download {'successful' if download_path else 'failed'}"
-        )
-        
-    except Exception as e:
-        if not cancel_flag.is_set():
-            logger.error_trace(f"Error in download processing: {e}")
-            book_queue.update_status(book_id, QueueStatus.ERROR)
-        else:
-            logger.info(f"Download cancelled: {book_id}")
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-
-def concurrent_download_loop() -> None:
-    """Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
-    logger.info(f"Starting concurrent download loop with {MAX_CONCURRENT_DOWNLOADS} workers")
-    
-    with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_DOWNLOADS, thread_name_prefix="BookDownload") as executor:
-        active_futures: Dict[Future, str] = {}  # Track active download futures
-        
-        while True:
-            # Clean up completed futures
-            completed_futures = [f for f in active_futures if f.done()]
-            for future in completed_futures:
-                book_id = active_futures.pop(future)
-                try:
-                    future.result()  # This will raise any exceptions from the worker
-                except Exception as e:
-                    logger.error_trace(f"Future exception for {book_id}: {e}")
-            
-            # Start new downloads if we have capacity
-            while len(active_futures) < MAX_CONCURRENT_DOWNLOADS:
-                next_download = book_queue.get_next()
-                if not next_download:
-                    break
-                    
-                book_id, cancel_flag = next_download
-                logger.info(f"Starting concurrent download: {book_id}")
-                
-                # Submit download job to thread pool
-                future = executor.submit(_process_single_download, book_id, cancel_flag)
-                active_futures[future] = book_id
-            
-            # Brief sleep to prevent busy waiting
-            time.sleep(MAIN_LOOP_SLEEP_TIME)
-
-# Start concurrent download coordinator
-download_coordinator_thread = threading.Thread(
-    target=concurrent_download_loop,
-    daemon=True,
-    name="DownloadCoordinator"
-)
-download_coordinator_thread.start()
-
-logger.info(f"Download system initialized with {MAX_CONCURRENT_DOWNLOADS} concurrent workers")

book_manager.py

@@ -1,381 +0,0 @@
-"""Book download manager handling search and retrieval operations."""
-
-import time, json, re
-from pathlib import Path
-from urllib.parse import quote
-from typing import List, Optional, Dict, Union
-from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet
-
-import downloader
-from logger import setup_logger
-from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL
-from env import AA_DONATOR_KEY, USE_CF_BYPASS, PRIORITIZE_WELIB
-from models import BookInfo, SearchFilters
-
-logger = setup_logger(__name__)
-
-
-def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
-    """Search for books matching the query.
-
-    Args:
-        query: Search term (ISBN, title, author, etc.)
-
-    Returns:
-        List[BookInfo]: List of matching books
-
-    Raises:
-        Exception: If no books found or parsing fails
-    """
-    query_html = quote(query)
-
-    if filters.isbn:
-        # ISBNs are included in query string
-        isbns = " || ".join(
-            [f"('isbn13:{isbn}' || 'isbn10:{isbn}')" for isbn in filters.isbn]
-        )
-        query_html = quote(f"({isbns}) {query}")
-
-    filters_query = ""
-
-    for value in filters.lang or BOOK_LANGUAGE:
-        if value != "all":
-            filters_query += f"&lang={quote(value)}"
-
-    if filters.sort:
-        filters_query += f"&sort={quote(filters.sort)}"
-
-    if filters.content:
-        for value in filters.content:
-            filters_query += f"&content={quote(value)}"
-
-    # Handle format filter
-    formats_to_use = filters.format if filters.format else SUPPORTED_FORMATS
-
-    index = 1
-    for filter_type, filter_values in vars(filters).items():
-        if filter_type == "author" or filter_type == "title" and filter_values:
-            for value in filter_values:
-                filters_query += (
-                    f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}"
-                )
-                index += 1
-
-    url = (
-        f"{AA_BASE_URL}"
-        f"/search?index=&page=1&display=table"
-        f"&acc=aa_download&acc=external_download"
-        f"&ext={'&ext='.join(formats_to_use)}"
-        f"&q={query_html}"
-        f"{filters_query}"
-    )
-
-    html = downloader.html_get_page(url)
-    if not html:
-        raise Exception("Failed to fetch search results")
-
-    if "No files found." in html:
-        logger.info(f"No books found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    soup = BeautifulSoup(html, "html.parser")
-    tbody: Tag | NavigableString | None = soup.find("table")
-
-    if not tbody:
-        logger.warning(f"No results table found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    books = []
-    if isinstance(tbody, Tag):
-        for line_tr in tbody.find_all("tr"):
-            try:
-                book = _parse_search_result_row(line_tr)
-                if book:
-                    books.append(book)
-            except Exception as e:
-                logger.error_trace(f"Failed to parse search result row: {e}")
-
-    books.sort(
-        key=lambda x: (
-            SUPPORTED_FORMATS.index(x.format)
-            if x.format in SUPPORTED_FORMATS
-            else len(SUPPORTED_FORMATS)
-        )
-    )
-
-    return books
-
-
-def _parse_search_result_row(row: Tag) -> Optional[BookInfo]:
-    """Parse a single search result row into a BookInfo object."""
-    try:
-        cells = row.find_all("td")
-        preview_img = cells[0].find("img")
-        preview = preview_img["src"] if preview_img else None
-
-        return BookInfo(
-            id=row.find_all("a")[0]["href"].split("/")[-1],
-            preview=preview,
-            title=cells[1].find("span").next,
-            author=cells[2].find("span").next,
-            publisher=cells[3].find("span").next,
-            year=cells[4].find("span").next,
-            language=cells[7].find("span").next,
-            format=cells[9].find("span").next.lower(),
-            size=cells[10].find("span").next,
-        )
-    except Exception as e:
-        logger.error_trace(f"Error parsing search result row: {e}")
-        return None
-
-
-def get_book_info(book_id: str) -> BookInfo:
-    """Get detailed information for a specific book.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-
-    Returns:
-        BookInfo: Detailed book information
-    """
-    url = f"{AA_BASE_URL}/md5/{book_id}"
-    html = downloader.html_get_page(url)
-
-    if not html:
-        raise Exception(f"Failed to fetch book info for ID: {book_id}")
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    return _parse_book_info_page(soup, book_id)
-
-
-def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
-    """Parse the book info page HTML into a BookInfo object."""
-    data = soup.select_one("body > main > div:nth-of-type(1)")
-
-    if not data:
-        raise Exception(f"Failed to parse book info for ID: {book_id}")
-
-    preview: str = ""
-
-    node = data.select_one("div:nth-of-type(1) > img")
-    if node:
-        preview_value = node.get("src", "")
-        if isinstance(preview_value, list):
-            preview = preview_value[0]
-        else:
-            preview = preview_value
-
-    data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
-    divs = list(data.children)
-    format = divs[13].text.split(" · ")[1].strip().lower()
-    size = divs[13].text.split(" · ")[2].strip().lower()
-
-    every_url = soup.find_all("a")
-    slow_urls_no_waitlist = set()
-    slow_urls_with_waitlist = set()
-    external_urls_libgen = set()
-    external_urls_z_lib = set()
-    external_urls_welib = set()
-
-    for url in every_url:
-        try:
-            if url.text.strip().lower().startswith("slow partner server"):
-                if (
-                    url.next is not None
-                    and url.next.next is not None
-                    and "waitlist" in url.next.next.strip().lower()
-                ):
-                    internal_text = url.next.next.strip().lower()
-                    if "no waitlist" in internal_text:
-                        slow_urls_no_waitlist.add(url["href"])
-                    else:
-                        slow_urls_with_waitlist.add(url["href"])
-            elif (
-                url.next is not None
-                and url.next.next is not None
-                and "click “GET” at the top" in url.next.next.text.strip()
-            ):
-                libgen_url = url["href"]
-                # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
-                libgen_url = libgen_url = re.sub(r'libgen\.(\w+)', 'libgen.gs', url["href"])
-                external_urls_libgen.add(libgen_url)
-            elif url.text.strip().lower().startswith("z-lib"):
-                if ".onion/" not in url["href"]:
-                    external_urls_z_lib.add(url["href"])
-        except:
-            pass
-
-    external_urls_welib = _get_download_urls_from_welib(book_id) if USE_CF_BYPASS else set()
-
-    urls = []
-    urls += list(external_urls_welib) if PRIORITIZE_WELIB else []
-    urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
-    urls += list(external_urls_libgen)
-    urls += list(external_urls_welib) if not PRIORITIZE_WELIB else []
-    urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
-    urls += list(external_urls_z_lib)
-
-    for i in range(len(urls)):
-        urls[i] = downloader.get_absolute_url(AA_BASE_URL, urls[i])
-
-    # Remove empty urls
-    urls = [url for url in urls if url != ""]
-
-    # Extract basic information
-    book_info = BookInfo(
-        id=book_id,
-        preview=preview,
-        title=divs[7].next.strip(),
-        publisher=divs[11].text.strip(),
-        author=divs[9].text.strip(),
-        format=format,
-        size=size,
-        download_urls=urls,
-    )
-
-    # Extract additional metadata
-    info = _extract_book_metadata(divs[-6])
-    book_info.info = info
-
-    # Set language and year from metadata if available
-    if info.get("Language"):
-        book_info.language = info["Language"][0]
-    if info.get("Year"):
-        book_info.year = info["Year"][0]
-
-    return book_info
-
-def _get_download_urls_from_welib(book_id: str) -> set[str]:
-    """Get download urls from welib.org."""
-    url = f"https://welib.org/md5/{book_id}"
-    logger.info(f"Getting download urls from welib.org for {book_id}. While this uses the bypasser, it will not start downloading them yet.")
-    html = downloader.html_get_page(url, use_bypasser=True)
-    if not html:
-        return []
-    soup = BeautifulSoup(html, "html.parser")
-    download_links = soup.find_all("a", href=True)
-    download_links = [link["href"] for link in download_links]
-    download_links = [link for link in download_links if "/slow_download/" in link]
-    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
-    return set(download_links)
-
-def _extract_book_metadata(
-    metadata_divs
-) -> Dict[str, List[str]]:
-    """Extract metadata from book info divs."""
-    info: Dict[str, List[str]] = {}
-
-    # Process the first set of metadata
-    sub_datas = metadata_divs.find_all("div")[0]
-    sub_datas = list(sub_datas.children)
-    for sub_data in sub_datas:
-        if sub_data.text.strip() == "":
-            continue
-        sub_data = list(sub_data.children)
-        key = sub_data[0].text.strip()
-        value = sub_data[1].text.strip()
-        if key not in info:
-            info[key] = set()
-        info[key].add(value)
-    
-    # make set into list
-    for key, value in info.items():
-        info[key] = list(value)
-
-    # Filter relevant metadata
-    relevant_prefixes = [
-        "ISBN-",
-        "ALTERNATIVE",
-        "ASIN",
-        "Goodreads",
-        "Language",
-        "Year",
-    ]
-    return {
-        k.strip(): v
-        for k, v in info.items()
-        if any(k.lower().startswith(prefix.lower()) for prefix in relevant_prefixes)
-        and "filename" not in k.lower()
-    }
-
-
-def download_book(book_info: BookInfo, book_path: Path) -> bool:
-    """Download a book from available sources.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-        title: Book title for logging
-
-    Returns:
-        Optional[BytesIO]: Book content buffer if successful
-    """
-
-    if len(book_info.download_urls) == 0:
-        book_info = get_book_info(book_info.id)
-    download_links = book_info.download_urls
-
-    # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources.
-    if AA_DONATOR_KEY != "":
-        download_links.insert(
-            0,
-            f"{AA_BASE_URL}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}",
-        )
-
-    for link in download_links:
-        try:
-            download_url = _get_download_url(link, book_info.title)
-            if download_url != "":
-                logger.info(f"Downloading `{book_info.title}` from `{download_url}`")
-                data = downloader.download_url(download_url, book_info.size or "")
-                if not data:
-                    raise Exception("No data received")
-
-                logger.info(f"Download finished. Writing to {book_path}")
-                with open(book_path, "wb") as f:
-                    f.write(data.getbuffer())
-                logger.info(f"Writing `{book_info.title}` successfully")
-                return True
-
-        except Exception as e:
-            logger.error_trace(f"Failed to download from {link}: {e}")
-            continue
-
-    return False
-
-
-def _get_download_url(link: str, title: str) -> str:
-    """Extract actual download URL from various source pages."""
-
-    url = ""
-
-    if link.startswith(f"{AA_BASE_URL}/dyn/api/fast_download.json"):
-        page = downloader.html_get_page(link)
-        url = json.loads(page).get("download_url")
-    else:
-        html = downloader.html_get_page(link)
-
-        if html == "":
-            return ""
-
-        soup = BeautifulSoup(html, "html.parser")
-
-        if link.startswith("https://z-lib."):
-            download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
-            if download_link:
-                url = download_link[0]["href"]
-        elif "/slow_download/" in link:
-            download_links = soup.find_all("a", href=True, string="📚 Download now")
-            if not download_links:
-                countdown = soup.find_all("span", class_="js-partner-countdown")
-                if countdown:
-                    sleep_time = int(countdown[0].text)
-                    logger.info(f"Waiting {sleep_time}s for {title}")
-                    time.sleep(sleep_time)
-                    url = _get_download_url(link, title)
-            else:
-                url = download_links[0]["href"]
-        else:
-            url = soup.find_all("a", string="GET")[0]["href"]
-
-    return downloader.get_absolute_url(link, url)

cloudflare_bypasser.py

@@ -1,478 +0,0 @@
-import time
-import os
-import socket
-from urllib.parse import urlparse
-import threading
-import env
-from env import LOG_DIR, DEBUG
-import signal
-from datetime import datetime
-import subprocess
-
-# --- SeleniumBase Import ---
-from seleniumbase import Driver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.common.exceptions import TimeoutException
-
-import network
-from logger import setup_logger
-from env import MAX_RETRY, DEFAULT_SLEEP
-from config import PROXIES, CUSTOM_DNS, DOH_SERVER, VIRTUAL_SCREEN_SIZE, RECORDING_DIR
-
-logger = setup_logger(__name__)
-network.init()
-
-DRIVER = None
-DISPLAY = {
-    "xvfb": None,
-    "ffmpeg": None,
-}
-LAST_USED = None
-LOCKED = threading.Lock()
-TENTATIVE_CURRENT_URL = None
-
-def _reset_pyautogui_display_state():
-    try:
-        import pyautogui
-        import Xlib.display
-        pyautogui._pyautogui_x11._display = (
-                    Xlib.display.Display(os.environ['DISPLAY'])
-                )
-    except Exception as e:
-        logger.warning(f"Error resetting pyautogui display state: {e}")
-
-def _is_bypassed(sb) -> bool:
-    """Enhanced bypass detection with more comprehensive checks"""
-    try:
-        # Get page information with error handling
-        try:
-            title = sb.get_title().lower()
-        except:
-            title = ""
-            
-        try:
-            body = sb.get_text("body").lower()
-        except:
-            body = ""
-            
-        try:
-            current_url = sb.get_current_url()
-        except:
-            current_url = ""
-        
-        # Enhanced verification texts for newer Cloudflare versions
-        verification_texts = [
-            "just a moment",
-            "verify you are human",
-            "verifying you are human",
-            "needs to review the security of your connection before proceeding",
-            "checking your browser",
-            "checking connection",
-            "attention required",
-            "access denied",
-            "needs to review the security of your connection",
-            "checking the site connection security",
-            "enable javascript and cookies to continue",
-            "ray id",
-            "cloudflare",
-            "please wait",
-            "ddos protection",
-            "security check",
-            "browser check",
-            "moment please",
-            "hold on",
-            "loading",
-            "one more step",
-            "challenge"
-        ]
-        
-        # Check for Cloudflare indicators
-        for text in verification_texts:
-            if text in title or text in body:
-                logger.debug(f"Cloudflare indicator found: '{text}' in page")
-                return False
-        
-        # Additional checks for specific Cloudflare patterns
-        if "cf-" in body or "cloudflare" in current_url.lower():
-            logger.debug("Cloudflare patterns detected in page")
-            return False
-            
-        # Check if we're still on a challenge page (common Cloudflare pattern)
-        if "/cdn-cgi/" in current_url:
-            logger.debug("Still on Cloudflare CDN challenge page")
-            return False
-            
-        # If page is mostly empty, it might still be loading
-        if len(body.strip()) < 50:
-            logger.debug("Page content too short, might still be loading")
-            return False
-            
-        logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {len(body)}")
-        return True
-        
-    except Exception as e:
-        logger.warning(f"Error checking bypass status: {e}")
-        # If we can't check, assume we're not bypassed
-        return False
-
-def _bypass_method_1(sb) -> bool:
-    """Original bypass method using uc_gui_click_captcha"""
-    try:
-        logger.debug("Attempting bypass method 1: uc_gui_click_captcha")
-        sb.uc_gui_click_captcha()
-        time.sleep(3)
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 1 failed on first try: {e}")
-        try:
-            time.sleep(5)
-            sb.wait_for_element_visible('body', timeout=10)
-            sb.uc_gui_click_captcha()
-            time.sleep(3)
-            return _is_bypassed(sb)
-        except Exception as e2:
-            logger.debug(f"Method 1 failed on second try: {e2}")
-            try:
-                time.sleep(DEFAULT_SLEEP)
-                sb.uc_gui_click_captcha()
-                time.sleep(5)
-                return _is_bypassed(sb)
-            except Exception as e3:
-                logger.debug(f"Method 1 completely failed: {e3}")
-                return False
-
-def _bypass_method_2(sb) -> bool:
-    """Alternative bypass method using longer waits and manual interaction"""
-    try:
-        logger.debug("Attempting bypass method 2: wait and reload")
-        # Wait longer for page to load completely
-        time.sleep(10)
-        
-        # Try refreshing the page
-        sb.refresh()
-        time.sleep(8)
-        
-        # Check if bypass worked after refresh
-        if _is_bypassed(sb):
-            return True
-            
-        # Try clicking on the page center (sometimes helps trigger bypass)
-        try:
-            sb.click_if_visible("body", timeout=5)
-            time.sleep(5)
-        except:
-            pass
-            
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 2 failed: {e}")
-        return False
-
-def _bypass_method_3(sb) -> bool:
-    """Third bypass method using user-agent rotation and stealth mode"""
-    try:
-        logger.debug("Attempting bypass method 3: stealth approach")
-        # Wait a random amount to appear more human
-        import random
-        wait_time = random.uniform(8, 15)
-        time.sleep(wait_time)
-        
-        # Try to scroll the page (human-like behavior)
-        try:
-            sb.scroll_to_bottom()
-            time.sleep(2)
-            sb.scroll_to_top()
-            time.sleep(3)
-        except:
-            pass
-            
-        # Check if this helped
-        if _is_bypassed(sb):
-            return True
-            
-        # Try the original captcha click as last resort
-        try:
-            sb.uc_gui_click_captcha()
-            time.sleep(5)
-        except:
-            pass
-            
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 3 failed: {e}")
-        return False
-
-def _bypass(sb, max_retries: int = MAX_RETRY) -> None:
-    """Enhanced bypass function with multiple strategies"""
-    try_count = 0
-    methods = [_bypass_method_1, _bypass_method_2, _bypass_method_3]
-
-    while not _is_bypassed(sb):
-        if try_count >= max_retries:
-            logger.warning("Exceeded maximum retries. Bypass failed.")
-            break
-            
-        method_index = try_count % len(methods)
-        method = methods[method_index]
-        
-        logger.info(f"Bypass attempt {try_count + 1} / {max_retries} using {method.__name__}")
-        
-        try_count += 1
-
-        # Progressive backoff: wait longer between retries
-        wait_time = min(DEFAULT_SLEEP * (try_count - 1), 15)
-        if wait_time > 0:
-            logger.info(f"Waiting {wait_time}s before trying...")
-            time.sleep(wait_time)
-
-        try:
-            if method(sb):
-                logger.info(f"Bypass successful using {method.__name__}")
-                return
-        except Exception as e:
-            logger.warning(f"Exception in {method.__name__}: {e}")
-
-        logger.info(f"Bypass method {method.__name__} failed.")
-
-def _get_chromium_args():
-    
-    arguments = [
-        # Ignore certificate and SSL errors (similar to curl's --insecure)
-        "--ignore-certificate-errors",
-        "--ignore-ssl-errors",
-        "--allow-running-insecure-content",
-        "--ignore-certificate-errors-spki-list",
-        "--ignore-certificate-errors-skip-list"
-    ]
-    
-    # Conditionally add verbose logging arguments
-    if DEBUG:
-        arguments.extend([
-            "--enable-logging", # Enable Chrome browser logging
-            "--v=1",            # Set verbosity level for Chrome logs
-            "--log-file=" + str(LOG_DIR / "chrome_browser.log")
-        ])
-
-    # Add proxy settings if configured
-    if PROXIES:
-        proxy_url = PROXIES.get('https') or PROXIES.get('http')
-        if proxy_url:
-            arguments.append(f'--proxy-server={proxy_url}')
-
-    # --- Add Custom DNS settings ---
-    try:
-        if len(CUSTOM_DNS) > 0:
-            if DOH_SERVER:
-                logger.info(f"Configuring DNS over HTTPS (DoH) with server: {DOH_SERVER}")
-
-                # TODO: This is probably broken and a halucination,
-                # but it should still default to google DOH so its fine...
-                arguments.extend(['--enable-features=DnsOverHttps', '--dns-over-https-mode=secure', f'--dns-over-https-servers="{DOH_SERVER}"'])
-                doh_hostname = urlparse(DOH_SERVER).hostname
-                if doh_hostname:
-                    try:
-                        arguments.append(f'--host-resolver-rules=MAP {doh_hostname} {socket.gethostbyname(doh_hostname)}')
-                    except socket.gaierror:
-                        logger.warning(f"Could not resolve DoH hostname: {doh_hostname}")
-            elif CUSTOM_DNS:
-                resolver_rules = [f"MAP * {dns_server}" for dns_server in CUSTOM_DNS]
-                if resolver_rules:
-                    arguments.append(f'--host-resolver-rules={",".join(resolver_rules)}') 
-    except Exception as e:
-        logger.error_trace(f"Error configuring DNS settings: {e}")
-    return arguments
-
-CHROMIUM_ARGS = _get_chromium_args()
-
-def _get(url, retry : int = MAX_RETRY):
-    try:
-        logger.info(f"SB_GET: {url}")
-        sb = _get_driver()
-        
-        # Enhanced page loading with better error handling
-        logger.debug("Opening URL with SeleniumBase...")
-        sb.uc_open_with_reconnect(url, DEFAULT_SLEEP)
-        time.sleep(DEFAULT_SLEEP)
-        
-        # Log current page title and URL for debugging
-        try:
-            current_url = sb.get_current_url()
-            current_title = sb.get_title()
-            logger.debug(f"Page loaded - URL: {current_url}, Title: {current_title}")
-        except Exception as debug_e:
-            logger.debug(f"Could not get page info: {debug_e}")
-        
-        # Attempt bypass
-        logger.debug("Starting bypass process...")
-        _bypass(sb)
-        
-        if _is_bypassed(sb):
-            logger.info("Bypass successful.")
-            return sb.page_source
-        else:
-            logger.warning("Bypass completed but page still shows Cloudflare protection")
-            # Log page content for debugging (truncated)
-            try:
-                page_text = sb.get_text("body")[:500] + "..." if len(sb.get_text("body")) > 500 else sb.get_text("body")
-                logger.debug(f"Page content: {page_text}")
-            except:
-                pass
-            
-    except Exception as e:
-        # Enhanced error logging with full stack trace
-        import traceback
-        error_details = f"Exception type: {type(e).__name__}, Message: {str(e)}"
-        stack_trace = traceback.format_exc()
-        
-        if retry == 0:
-            logger.error(f"Failed to initialize browser after all retries: {error_details}")
-            logger.debug(f"Full stack trace: {stack_trace}")
-            _reset_driver()
-            raise e
-        
-        logger.warning(f"Failed to bypass Cloudflare (retry {MAX_RETRY - retry + 1}/{MAX_RETRY}): {error_details}")
-        logger.debug(f"Stack trace: {stack_trace}")
-        
-        # Reset driver on certain errors
-        if "WebDriverException" in str(type(e)) or "SessionNotCreatedException" in str(type(e)):
-            logger.info("Resetting driver due to WebDriver error...")
-            _reset_driver()
-            
-    return _get(url, retry - 1)
-
-def get(url, retry : int = MAX_RETRY):
-    global LOCKED, TENTATIVE_CURRENT_URL, LAST_USED
-    with LOCKED:
-        TENTATIVE_CURRENT_URL = url
-        ret = _get(url, retry)
-        LAST_USED = time.time()
-        return ret
-
-def _init_driver():
-    global DRIVER
-    if DRIVER:
-        _reset_driver()
-    driver = Driver(uc=True, headless=False, size=f"{VIRTUAL_SCREEN_SIZE[0]},{VIRTUAL_SCREEN_SIZE[1]}", chromium_arg=CHROMIUM_ARGS)
-    DRIVER = driver
-    time.sleep(DEFAULT_SLEEP)
-    return driver
-
-def _get_driver():
-    global DRIVER, DISPLAY
-    global LAST_USED
-    logger.info("Getting driver...")
-    LAST_USED = time.time()
-    if env.DOCKERMODE and env.USE_CF_BYPASS and not DISPLAY["xvfb"]:
-        from pyvirtualdisplay import Display
-        display = Display(visible=False, size=VIRTUAL_SCREEN_SIZE)
-        display.start()
-        logger.info("Display started")
-        DISPLAY["xvfb"] = display
-        time.sleep(DEFAULT_SLEEP)
-        _reset_pyautogui_display_state()
-
-        if env.DEBUG:
-            timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
-            output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
-
-            ffmpeg_cmd = [
-                "ffmpeg",
-                "-y",
-                "-f", "x11grab",
-                "-video_size", f"{VIRTUAL_SCREEN_SIZE[0]}x{VIRTUAL_SCREEN_SIZE[1]}",
-                "-i", f":{display.display}",
-                "-c:v", "libx264",
-                "-preset", "ultrafast",  # or "veryfast" (trade speed for slightly better compression)
-                "-maxrate", "700k",      # Slightly higher bitrate for text clarity
-                "-bufsize", "1400k",    # Buffer size (2x maxrate)
-                "-crf", "36",  # Adjust as needed:  higher = smaller, lower = better quality (23 is visually lossless)
-                "-pix_fmt", "yuv420p",  # Crucial for compatibility with most players
-                "-tune", "animation",   # Optimize encoding for screen content
-                "-x264-params", "bframes=0:deblock=-1,-1", # Optimize for text, disable b-frames and deblocking
-                "-r", "15",         # Reduce frame rate (if content allows)
-                "-an",                # Disable audio recording (if not needed)
-                output_file.as_posix(),
-                "-nostats", "-loglevel", "0"
-            ]
-            logger.info("Starting FFmpeg recording to %s", output_file)
-            logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
-            DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
-    if not DRIVER:
-        return _init_driver()
-    logger.log_resource_usage()
-    return DRIVER
-
-def _reset_driver():
-    logger.log_resource_usage()
-    logger.info("Resetting driver...")
-    global DRIVER, DISPLAY
-    if DRIVER:
-        try:
-            DRIVER.quit()
-            DRIVER = None
-        except Exception as e:
-            logger.warning(f"Error quitting driver: {e}")
-        time.sleep(0.5)
-    if DISPLAY["xvfb"]:
-        try:
-            DISPLAY["xvfb"].stop()
-            DISPLAY["xvfb"] = None
-        except Exception as e:
-            logger.warning(f"Error stopping display: {e}")
-        time.sleep(0.5)
-    try:
-        os.system("pkill -f Xvfb")
-    except Exception as e:
-        logger.debug(f"Error killing Xvfb: {e}")
-    time.sleep(0.5)
-    if DISPLAY["ffmpeg"]:
-        try:
-            DISPLAY["ffmpeg"].send_signal(signal.SIGINT)
-            DISPLAY["ffmpeg"] = None
-        except Exception as e:
-            logger.debug(f"Error stopping ffmpeg: {e}")
-        time.sleep(0.5)
-    try:
-        os.system("pkill -f ffmpeg")
-    except Exception as e:
-        logger.debug(f"Error killing ffmpeg: {e}")
-    time.sleep(0.5)
-    try:
-        os.system("pkill -f chrom")
-    except Exception as e:
-        logger.debug(f"Error killing chrom: {e}")
-    time.sleep(0.5)
-    logger.info("Driver reset.")
-    logger.log_resource_usage()
-
-def _cleanup_driver():
-    global LOCKED
-    global LAST_USED
-    with LOCKED:
-        if LAST_USED:
-            if time.time() - LAST_USED >= env.BYPASS_RELEASE_INACTIVE_MIN * 60:
-                _reset_driver()
-                LAST_USED = None
-                logger.info("Driver reset due to inactivity.")
-
-def _cleanup_loop():
-    while True:
-        _cleanup_driver()
-        time.sleep(max(env.BYPASS_RELEASE_INACTIVE_MIN / 2, 1))
-
-def _init_cleanup_thread():
-    cleanup_thread = threading.Thread(target=_cleanup_loop)
-    cleanup_thread.daemon = True
-    cleanup_thread.start()
-
-def wait_for_result(func, timeout : int = 10, condition : any = True):
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        result = func()
-        if condition(result):
-            return result
-        time.sleep(0.5)
-    return None
-_init_cleanup_thread()

compose/docker-compose.lite.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark-lite:
+    image: ghcr.io/calibrain/shelfmark-lite:latest
+    environment:
+      # EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.tor.yml

@@ -0,0 +1,20 @@
+# Routes all traffic through Tor - requires NET_ADMIN capability
+services:
+  shelfmark-tor:
+    image: ghcr.io/calibrain/shelfmark:latest
+    environment:
+      FLASK_PORT: 8084
+      USING_TOR: true
+      PUID: 1000
+      PGID: 1000
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    container_name: shelfmark
+    environment:
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

config.py

@@ -1,99 +0,0 @@
-"""Configuration settings for the book downloader application."""
-
-import os
-from pathlib import Path
-import json
-import env
-from logger import setup_logger
-
-logger = setup_logger(__name__)
-
-for key, value in env.__dict__.items():
-    if not key.startswith('_'):
-        if key == "AA_DONATOR_KEY" and value.strip() != "":
-            value = "REDACTED"
-        logger.info(f"{key}: {value}")
-
-with open("data/book-languages.json") as file:
-    _SUPPORTED_BOOK_LANGUAGE = json.load(file)
-
-# Directory settings
-BASE_DIR = Path(__file__).resolve().parent
-logger.info(f"BASE_DIR: {BASE_DIR}")
-if env.ENABLE_LOGGING:
-    env.LOG_DIR.mkdir(exist_ok=True)
-
-# Create necessary directories
-env.TMP_DIR.mkdir(exist_ok=True)
-env.INGEST_DIR.mkdir(exist_ok=True)
-
-CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
-logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
-logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
-logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
-
-# Network settings
-_custom_dns = env._CUSTOM_DNS.lower().strip()
-_doh_server = ""
-if _custom_dns == "google":
-    CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860::8888", "2001:4860:4860::8844"]
-    _doh_server = "https://dns.google/dns-query"
-elif _custom_dns == "quad9":
-    CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:fe::fe", "26620:fe::9"]
-    _doh_server = "https://dns.quad9.net/dns-query"
-elif _custom_dns == "cloudflare":
-    CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700::1111", "2606:4700:4700::1001"]
-    _doh_server = "https://cloudflare-dns.com/dns-query"
-elif _custom_dns == "opendns":
-    CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:119:35::35", "2620:119:53::53"]
-    _doh_server = "https://doh.opendns.com/dns-query"
-else:
-    _custom_dns_ip = _custom_dns.split(",")
-    CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()]
-logger.info(f"CUSTOM_DNS: {CUSTOM_DNS}")
-DOH_SERVER = _doh_server
-if env.USE_DOH:
-    DOH_SERVER = _doh_server
-else:
-    DOH_SERVER = ""
-logger.info(f"DOH_SERVER: {DOH_SERVER}")
-
-# Proxy settings
-PROXIES = {}
-if env.HTTP_PROXY:
-    PROXIES["http"] = env.HTTP_PROXY
-if env.HTTPS_PROXY:
-    PROXIES["https"] = env.HTTPS_PROXY
-logger.info(f"PROXIES: {PROXIES}")
-
-# Anna's Archive settings
-AA_BASE_URL = env._AA_BASE_URL
-AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
-AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
-AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
-
-# File format settings
-SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
-logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
-
-# Complex language processing logic kept in config.py
-BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
-BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
-if len(BOOK_LANGUAGE) == 0:
-    BOOK_LANGUAGE = ['en']
-
-# Custom script settings with validation logic
-CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
-if CUSTOM_SCRIPT:
-    if not os.path.exists(CUSTOM_SCRIPT):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
-        CUSTOM_SCRIPT = ""
-    elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
-        CUSTOM_SCRIPT = ""
-
-# Debugging settings
-VIRTUAL_SCREEN_SIZE = (1024, 768)
-RECORDING_DIR = env.LOG_DIR / "recording"
-if env.DEBUG:
-    RECORDING_DIR.mkdir(parents=True, exist_ok=True)

docker-compose.dev.extbp.yml

@@ -0,0 +1,25 @@
+# Local development - External bypasser variant (lite)
+services:
+  shelfmark-lite-dev:
+    extends:
+      file: ./compose/docker-compose.lite.yml
+      service: shelfmark-lite
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark-lite
+    environment:
+      DEBUG: true
+      EXT_BYPASSER_URL: http://flaresolverr:8191
+      EXT_BYPASSER_PATH: /v1
+      EXT_BYPASSER_TIMEOUT: 60000
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads
+
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.dev.tor.yml

@@ -0,0 +1,20 @@
+# Local development - Tor variant
+services:
+  shelfmark-tor-dev:
+    extends:
+      file: ./compose/docker-compose.tor.yml
+      service: shelfmark-tor
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    environment:
+      DEBUG: true
+      USING_TOR: true
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.dev.yml

@@ -1,17 +1,22 @@
+# Local development - builds from source with debug enabled
 services:
-  calibre-web-automated-book-downloader-dev:
+  shelfmark-dev:
     extends:
-      file: ./docker-compose.yml
-      service: calibre-web-automated-book-downloader
+      file: ./compose/docker-compose.yml
+      service: shelfmark
     build:
       context: .
       dockerfile: Dockerfile
-      target: cwa-bd
+      target: shelfmark
+    cap_add:
+      - SYS_PTRACE
     environment:
       DEBUG: true
-      APP_ENV: dev
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
     volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      - ./shelfmark:/app/shelfmark:ro
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.test-clients.yml

@@ -0,0 +1,181 @@
+# Test stack for download client development
+# Includes shelfmark + all download clients on same network with shared volumes
+#
+# Usage:
+#   docker compose -f docker-compose.test-clients.yml up -d
+#   # Access shelfmark at http://localhost:8084
+#   # Configure clients in Settings > Prowlarr > Download Clients
+#
+# Web UIs:
+#   - shelfmark:    http://localhost:8084
+#   - Prowlarr:     http://localhost:9696 (no auth by default)
+#   - qBittorrent:  http://localhost:8080 (check container logs for temp password)
+#   - Transmission: http://localhost:9091 (admin / admin)
+#   - Deluge:       http://localhost:8112 (password: deluge)
+#   - NZBGet:       http://localhost:6789 (nzbget / tegbzn6789)
+#   - SABnzbd:      http://localhost:8085 (complete setup wizard for API key)
+#   - rTorrent:     http://localhost:8000 (admin / admin - if auth enabled)
+#
+
+services:
+  shelfmark:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    container_name: test-shelfmark
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      # All client configuration is done via Settings UI
+      # Use Docker service names for URLs:
+      #   - qBittorrent: http://qbittorrent:8080
+      #   - Transmission: http://transmission:9091
+      #   - Deluge Web UI: http://deluge:8112
+      #   - NZBGet: http://nzbget:6789
+      #   - SABnzbd: http://sabnzbd:8080
+      #   - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
+    ports:
+      - "8084:8084"
+    volumes:
+      # Config and state
+      - ./.local/test-clients/shelfmark/config:/config
+      - ./.local/test-clients/shelfmark/log:/var/log/shelfmark
+      # Book destination directory (where completed books go)
+      - ./.local/test-clients/books:/books
+      # Staging directory
+      - ./.local/test-clients/tmp:/tmp/shelfmark
+      # CRITICAL: Mount client download directories so shelfmark can access completed files
+      - ./.local/test-clients/downloads:/downloads
+      # Mount source code for hot-reload (no rebuild needed for Python changes)
+      - ./shelfmark:/app/shelfmark:ro
+      # Mount tests for running pytest in container
+      - ./tests:/app/tests:ro
+      - ./pyproject.toml:/app/pyproject.toml:ro
+      # Mount client configs for integration tests to read credentials
+      - ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
+      - ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
+    depends_on:
+      - nzbget
+      - sabnzbd
+      - qbittorrent
+      - transmission
+      - deluge
+      - rtorrent
+    restart: unless-stopped
+
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:latest
+    container_name: test-prowlarr
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/prowlarr/config:/config
+    ports:
+      - "9696:9696"
+    restart: unless-stopped
+
+  nzbget:
+    image: lscr.io/linuxserver/nzbget:latest
+    container_name: test-nzbget
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/nzbget/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/nzbget/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "6789:6789"  # Web UI / JSON-RPC
+    restart: unless-stopped
+
+  sabnzbd:
+    image: lscr.io/linuxserver/sabnzbd:latest
+    container_name: test-sabnzbd
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/sabnzbd/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8085:8080"  # Web UI (external:internal)
+    restart: unless-stopped
+
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: test-qbittorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - WEBUI_PORT=8080
+    volumes:
+      - ./.local/test-clients/qbittorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/qbittorrent/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "8080:8080"  # Web UI / API
+      - "6882:6881"
+      - "6882:6881/udp"
+    restart: unless-stopped
+
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: test-transmission
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - USER=admin
+      - PASS=admin
+    volumes:
+      - ./.local/test-clients/transmission/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "9091:9091"  # Web UI / RPC
+      - "51413:51413"
+      - "51413:51413/udp"
+    restart: unless-stopped
+
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: test-deluge
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - DELUGE_LOGLEVEL=error
+    volumes:
+      - ./.local/test-clients/deluge/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8112:8112"  # Web UI
+      - "58846:58846"  # Daemon RPC
+      - "6881:6881"
+      - "6881:6881/udp"
+    restart: unless-stopped
+
+  rtorrent:
+    image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
+    container_name: test-rtorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/rtorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8000:8000"  # XMLRPC
+      - "8089:8080"  # ruTorrent Web UI
+      - "9000:9000"  # SCGI port
+      - "50000:50000"  # Incoming connections
+      - "6881:6881/udp"
+    restart: unless-stopped

docker-compose.tor.dev.yml

@@ -1,15 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor-dev:
-    extends:
-      file: ./docker-compose.tor.yml
-      service: calibre-web-automated-book-downloader-tor
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-tor
-    environment:
-      DEBUG: true
-      APP_ENV: dev
-    volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader

docker-compose.tor.yml

@@ -1,21 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      USING_TOR: true
-      APP_ENV: prod
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-    # This is where the books will be downloaded to, usually it would be 
-    # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest

docker-compose.yml

@@ -1,30 +0,0 @@
-services:
-  calibre-web-automated-book-downloader:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
-    # Uncomment to build the image from the Dockerfile for local testing changes.
-    # Remember to comment out the image line above.
-    #build: .
-    container_name: calibre-web-automated-book-downloader
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      APP_ENV: prod
-      UID: 1000
-      GID: 100
-      # CWA_DB_PATH: /auth/app.db  # Comment out to disable authentication
-      # Queue management settings
-      MAX_CONCURRENT_DOWNLOADS: 3
-      DOWNLOAD_PROGRESS_UPDATE_INTERVAL: 5
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Comment out to disable authentication
-      #- /cwa/config/path/app.db:/auth/app.db:ro

docs/configuration.md

@@ -0,0 +1,3 @@
+# Configuration
+
+TODO

docs/dev/index.md

@@ -0,0 +1,3 @@
+# Developer Documentation
+
+TODO

docs/dev/plugin-settings.md

@@ -0,0 +1,628 @@
+# Plugin Settings Integration Guide
+
+This guide explains how to add configuration settings to plugins (Metadata Providers and Release Sources) so they appear in the Settings UI.
+
+## Overview
+
+The settings system uses a decorator-based registration pattern. Plugins register their settings when their module is imported, and the frontend dynamically renders the appropriate UI based on the schema provided by the backend.
+
+**Key features:**
+- Settings are defined in Python and automatically rendered in the React frontend
+- Values persist across container restarts via JSON config files
+- Changes take effect immediately without restart (unless marked otherwise)
+
+## Quick Start
+
+Add settings to your plugin in 3 steps:
+
+```python
+from shelfmark.core.settings_registry import (
+    register_settings,
+    TextField,
+    PasswordField,
+    ActionButton,
+)
+
+@register_settings(
+    name="my_plugin",           # Unique identifier
+    display_name="My Plugin",   # Shown in sidebar
+    icon="wrench",              # Icon name
+    order=100,                  # Sort order (lower = higher in list)
+    group="metadata_providers"  # Optional: group in sidebar
+)
+def my_plugin_settings():
+    return [
+        PasswordField(
+            key="MY_PLUGIN_API_KEY",
+            label="API Key",
+            description="Your API key from the provider",
+            required=True,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+def _test_connection():
+    # Perform connection test
+    return {"success": True, "message": "Connected successfully!"}
+```
+
+## Available Field Types
+
+### TextField
+
+Single-line text input for strings.
+
+```python
+TextField(
+    key="MY_SETTING",           # Config key
+    label="Setting Name",       # Display label
+    description="Help text",    # Optional description below field
+    default="",                 # Default value
+    placeholder="Enter value",  # Placeholder text
+    max_length=100,             # Optional max characters
+    required=False,             # Is this field required?
+    requires_restart=False,     # Does changing this need a restart?
+    show_when=None,             # Conditional visibility (see below)
+    disabled_when=None,         # Conditional disable (see below)
+)
+```
+
+### PasswordField
+
+Masked input for sensitive values (API keys, passwords). Values are never echoed back to the frontend.
+
+```python
+PasswordField(
+    key="API_KEY",
+    label="API Key",
+    description="Your secret API key",
+    placeholder="sk-...",
+    required=True,
+)
+```
+
+### NumberField
+
+Numeric input with optional min/max constraints.
+
+```python
+NumberField(
+    key="TIMEOUT",
+    label="Timeout (seconds)",
+    description="Connection timeout in seconds",
+    default=30,
+    min_value=5,
+    max_value=300,
+    step=1,                     # Increment step
+    required=False,
+)
+```
+
+### CheckboxField
+
+Toggle switch for boolean values.
+
+```python
+CheckboxField(
+    key="ENABLE_FEATURE",
+    label="Enable Feature",
+    description="Turn this feature on or off",
+    default=False,
+)
+```
+
+### SelectField
+
+Dropdown for single-choice selection.
+
+```python
+SelectField(
+    key="LOG_LEVEL",
+    label="Log Level",
+    description="Logging verbosity",
+    default="info",
+    options=[
+        {"value": "debug", "label": "Debug"},
+        {"value": "info", "label": "Info"},
+        {"value": "warning", "label": "Warning"},
+        {"value": "error", "label": "Error"},
+    ],
+)
+```
+
+### MultiSelectField
+
+Multi-choice selection from a list of options.
+
+```python
+MultiSelectField(
+    key="SUPPORTED_FORMATS",
+    label="Supported Formats",
+    description="Select which formats to support",
+    default=["epub", "mobi"],
+    options=[
+        {"value": "epub", "label": "EPUB"},
+        {"value": "mobi", "label": "MOBI"},
+        {"value": "pdf", "label": "PDF"},
+        {"value": "azw3", "label": "AZW3"},
+    ],
+)
+```
+
+### ActionButton
+
+Button that executes a callback function. Does not store a value.
+
+```python
+ActionButton(
+    key="test_connection",      # Unique key for the action
+    label="Test Connection",    # Button text
+    description="Test the API connection",
+    style="primary",            # "default", "primary", or "danger"
+    callback=my_callback_fn,    # Function to execute
+)
+
+def my_callback_fn():
+    """Callback must return dict with 'success' and 'message' keys."""
+    try:
+        # Perform action
+        return {"success": True, "message": "Connection successful!"}
+    except Exception as e:
+        return {"success": False, "message": f"Failed: {str(e)}"}
+```
+
+### HeadingField
+
+Display-only section heading with optional link. Does not store a value.
+
+```python
+HeadingField(
+    key="section_heading",      # Unique key
+    title="Configuration",      # Heading text
+    description="Configure the plugin settings below",
+    link_url="https://example.com/docs",  # Optional link
+    link_text="View Documentation",       # Link text
+)
+```
+
+## Common Field Properties
+
+All field types support these common properties:
+
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `key` | `str` | Required | Unique identifier for this setting |
+| `label` | `str` | Required | Display label in the UI |
+| `description` | `str` | `""` | Help text shown below the field |
+| `default` | `Any` | `None` | Default value if not set |
+| `required` | `bool` | `False` | Whether the field must have a value |
+| `disabled` | `bool` | `False` | Disable the field (greyed out) |
+| `disabled_reason` | `str` | `""` | Explanation shown when disabled |
+| `requires_restart` | `bool` | `False` | Whether changes require container restart |
+| `show_when` | `dict` | `None` | Conditional visibility (see below) |
+| `disabled_when` | `dict` | `None` | Conditional disable (see below) |
+
+## Conditional Visibility
+
+Fields can be shown/hidden based on other field values using `show_when`:
+
+```python
+# Only show DNS servers field when custom DNS is selected
+TextField(
+    key="CUSTOM_DNS_SERVERS",
+    label="DNS Servers",
+    description="Comma-separated DNS server IPs",
+    show_when={"field": "DNS_PROVIDER", "value": "manual"},
+)
+```
+
+The field will only be visible when the referenced field has the specified value.
+
+## Conditional Disable
+
+Fields can be enabled/disabled based on other field values using `disabled_when`:
+
+```python
+# Disable timeout field when feature is disabled
+NumberField(
+    key="FEATURE_TIMEOUT",
+    label="Timeout (seconds)",
+    description="Request timeout",
+    default=30,
+    disabled_when={
+        "field": "FEATURE_ENABLED",
+        "value": False,
+        "reason": "Enable the feature first"
+    },
+)
+```
+
+The field will be greyed out with the specified reason when the condition is met.
+
+## Settings Groups
+
+Register a group to organize related settings tabs in the sidebar:
+
+```python
+from shelfmark.core.settings_registry import register_group
+
+# Register a group (do this once, usually in a central config file)
+register_group(
+    name="my_group",
+    display_name="My Group",
+    icon="folder",
+    order=50,
+)
+
+# Then register settings to the group
+@register_settings(
+    name="plugin_a",
+    display_name="Plugin A",
+    icon="puzzle",
+    order=51,
+    group="my_group",  # Assigns to the group
+)
+def plugin_a_settings():
+    return [...]
+```
+
+**Existing groups:**
+- `direct_download` (order=20): For download-related settings
+- `metadata_providers` (order=50): For metadata provider plugins
+
+## Value Resolution Priority
+
+Settings values are resolved in this order (highest priority first):
+
+1. **Config File** - Stored in `CONFIG_DIR/plugins/<tab_name>.json`
+2. **Field Default** - Value specified in the field definition
+
+The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdirectory.
+
+## Reading Setting Values
+
+Use the `config` singleton to read setting values in your plugin code:
+
+```python
+from shelfmark.core.config import config
+
+# Get a setting value with default fallback
+api_key = config.get("MY_PLUGIN_API_KEY", "")
+timeout = config.get("MY_PLUGIN_TIMEOUT", 30)
+
+# Or access as attributes (raises AttributeError if not found)
+api_key = config.MY_PLUGIN_API_KEY
+
+# Check all cached settings
+all_settings = config.get_all()
+```
+
+The config singleton:
+- Automatically resolves values from config files with field defaults as fallback
+- Caches values for performance
+- Refreshes automatically when settings are updated via the UI
+
+## Complete Example: Metadata Provider
+
+Here's a complete example for a metadata provider plugin:
+
+```python
+# shelfmark/metadata_providers/my_provider.py
+
+from shelfmark.metadata_providers.base import (
+    MetadataProvider,
+    register_provider,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_connection():
+    """Test API connection callback."""
+    api_key = config.get("MY_PROVIDER_API_KEY", "")
+    if not api_key:
+        return {"success": False, "message": "API key not configured"}
+
+    try:
+        # Perform actual connection test
+        # response = requests.get(...)
+        return {"success": True, "message": "Connected to My Provider API"}
+    except Exception as e:
+        return {"success": False, "message": f"Connection failed: {str(e)}"}
+
+
+@register_settings(
+    name="my_provider",
+    display_name="My Provider",
+    icon="book",
+    order=53,
+    group="metadata_providers",
+)
+def my_provider_settings():
+    """Define settings for this metadata provider."""
+    return [
+        HeadingField(
+            key="my_provider_heading",
+            title="My Provider",
+            description="A metadata provider for book information",
+            link_url="https://myprovider.com",
+            link_text="Visit My Provider",
+        ),
+        PasswordField(
+            key="MY_PROVIDER_API_KEY",
+            label="API Key",
+            description="Your My Provider API key",
+            placeholder="Enter your API key",
+            required=True,
+        ),
+        CheckboxField(
+            key="MY_PROVIDER_INCLUDE_COVERS",
+            label="Include Cover Images",
+            description="Fetch cover images when searching",
+            default=True,
+        ),
+        TextField(
+            key="MY_PROVIDER_BASE_URL",
+            label="API Base URL",
+            description="Override the default API endpoint",
+            default="https://api.myprovider.com/v1",
+            required=False,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            description="Verify your API key works",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+
+@register_provider("my_provider")
+class MyProvider(MetadataProvider):
+    """My Provider metadata implementation."""
+
+    name = "my_provider"
+    display_name = "My Provider"
+    requires_auth = True
+
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
+        self.base_url = config.get(
+            "MY_PROVIDER_BASE_URL",
+            "https://api.myprovider.com/v1"
+        )
+
+    def is_available(self) -> bool:
+        return bool(self.api_key)
+
+    def search(self, query: str):
+        # Implementation...
+        pass
+
+    def get_book(self, book_id: str):
+        # Implementation...
+        pass
+```
+
+## Complete Example: Release Source
+
+Here's a complete example for a release source plugin:
+
+```python
+# shelfmark/release_sources/my_source.py
+
+from shelfmark.release_sources.base import (
+    ReleaseSource,
+    DownloadHandler,
+    register_source,
+    register_handler,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    NumberField,
+    CheckboxField,
+    SelectField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_source():
+    """Test source availability callback."""
+    base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+    try:
+        # Test connectivity
+        return {"success": True, "message": f"Source available at {base_url}"}
+    except Exception as e:
+        return {"success": False, "message": f"Source unavailable: {str(e)}"}
+
+
+@register_settings(
+    name="my_source",
+    display_name="My Source",
+    icon="download",
+    order=25,
+    group="direct_download",
+)
+def my_source_settings():
+    """Define settings for this release source."""
+    return [
+        HeadingField(
+            key="my_source_heading",
+            title="My Source Configuration",
+            description="Configure the My Source download provider",
+        ),
+        CheckboxField(
+            key="MY_SOURCE_ENABLED",
+            label="Enable My Source",
+            description="Include My Source in download fallback chain",
+            default=True,
+        ),
+        TextField(
+            key="MY_SOURCE_URL",
+            label="Source URL",
+            description="Base URL for the source",
+            default="https://mysource.com",
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        NumberField(
+            key="MY_SOURCE_TIMEOUT",
+            label="Timeout (seconds)",
+            description="Request timeout",
+            default=30,
+            min_value=10,
+            max_value=120,
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        SelectField(
+            key="MY_SOURCE_PRIORITY",
+            label="Priority",
+            description="Where in the fallback chain to try this source",
+            default="normal",
+            options=[
+                {"value": "high", "label": "High (try first)"},
+                {"value": "normal", "label": "Normal"},
+                {"value": "low", "label": "Low (try last)"},
+            ],
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        ActionButton(
+            key="test_source",
+            label="Test Source",
+            description="Check if the source is accessible",
+            style="primary",
+            callback=_test_source,
+        ),
+    ]
+
+
+@register_source("my_source")
+class MySource(ReleaseSource):
+    """My Source release source implementation."""
+
+    name = "my_source"
+    display_name = "My Source"
+
+    def __init__(self):
+        self.enabled = config.get("MY_SOURCE_ENABLED", True)
+        self.base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+        self.timeout = config.get("MY_SOURCE_TIMEOUT", 30)
+
+    def is_available(self) -> bool:
+        return self.enabled
+
+    def search(self, book):
+        # Implementation...
+        pass
+
+
+@register_handler("my_source")
+class MySourceHandler(DownloadHandler):
+    """Handler for downloading from My Source."""
+
+    name = "my_source"
+
+    def download(self, release, output_path):
+        # Implementation...
+        pass
+```
+
+## Best Practices
+
+1. **Use descriptive keys**: Keys should be uppercase and prefixed with your plugin name (e.g., `MY_PLUGIN_API_KEY`)
+
+2. **Provide helpful descriptions**: Include enough detail in descriptions to help users understand what each setting does
+
+3. **Set sensible defaults**: Users should be able to get started without configuring everything
+
+4. **Use conditional visibility**: Hide advanced options behind enabling checkboxes to reduce UI clutter
+
+5. **Include a test button**: ActionButtons that test connections help users verify their configuration
+
+6. **Mark restart-required settings**: Use `requires_restart=True` for settings that can't be applied live
+
+7. **Group related settings**: Use HeadingField to visually separate sections, and put plugins in appropriate groups
+
+8. **Handle missing values gracefully**: Always provide fallbacks when reading settings in your code
+
+## API Reference
+
+### Backend Routes
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/api/settings` | Get all settings tabs, groups, and values |
+| GET | `/api/settings/<tab_name>` | Get a specific settings tab |
+| PUT | `/api/settings/<tab_name>` | Update settings for a tab |
+| POST | `/api/settings/<tab_name>/action/<action_key>` | Execute an action button callback |
+
+### Response Format
+
+**GET /api/settings**
+```json
+{
+  "groups": [
+    {"name": "direct_download", "displayName": "Direct Download", "icon": "download", "order": 20}
+  ],
+  "tabs": [
+    {
+      "name": "my_plugin",
+      "displayName": "My Plugin",
+      "icon": "book",
+      "order": 53,
+      "group": "metadata_providers",
+      "fields": [
+        {
+          "type": "password",
+          "key": "MY_PLUGIN_API_KEY",
+          "label": "API Key",
+          "description": "Your API key",
+          "hasValue": true,
+          "value": "",
+          "required": true,
+          "disabled": false,
+          "requiresRestart": false
+        }
+      ]
+    }
+  ]
+}
+```
+
+**PUT /api/settings/<tab_name>**
+```json
+// Request
+{"MY_PLUGIN_API_KEY": "new-value", "MY_PLUGIN_TIMEOUT": 60}
+
+// Response
+{
+  "success": true,
+  "message": "Settings updated",
+  "updated": ["MY_PLUGIN_API_KEY", "MY_PLUGIN_TIMEOUT"],
+  "requiresRestart": false
+}
+```
+
+**POST /api/settings/<tab_name>/action/<action_key>**
+```json
+// Response
+{
+  "success": true,
+  "message": "Connection successful!"
+}
+```

docs/index.md

@@ -0,0 +1,3 @@
+# Shelfmark Documentation
+
+TODO

docs/installation.md

@@ -0,0 +1,3 @@
+# Installation
+
+TODO

docs/reverse-proxy.md

@@ -0,0 +1,35 @@
+# Reverse Proxy & Subpath Hosting
+
+Shelfmark can run behind a reverse proxy at the root path (recommended) or
+under a subpath like `/shelfmark`.
+
+## Subpath setup
+
+1) Set the base path in Shelfmark:
+- UI: Settings → Advanced → Base Path
+- Env var: `URL_BASE=/shelfmark`
+
+2) Configure your reverse proxy to forward the subpath to Shelfmark and
+**strip the prefix** before sending to the backend. The proxy must also allow
+WebSocket upgrades for Socket.IO.
+
+Example (Nginx-style):
+
+```
+location /shelfmark/ {
+  proxy_pass http://shelfmark:8084/;
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection "upgrade";
+}
+```
+
+Notes:
+- Use a trailing slash on the `location` and `proxy_pass` to ensure the
+  `/shelfmark` prefix is removed.
+- Health checks still work at `/api/health` without the subpath.
+
+## Root path setup
+
+If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`),
+leave `URL_BASE` empty. This is the simplest option.

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+# Troubleshooting
+
+TODO

docs/url-search-parameters.md

@@ -0,0 +1,75 @@
+# URL Search Parameters
+
+You can trigger searches directly via URL by adding query parameters. This enables bookmarking searches and sharing links.
+
+## Basic Usage
+
+```
+http://your-server:8084/?q=harry+potter
+```
+
+## Supported Parameters
+
+| Parameter | Description | Example |
+|-----------|-------------|---------|
+| `q` or `query` | Main search query | `/?q=dune` |
+| `author` | Filter by author name | `/?author=frank+herbert` |
+| `title` | Filter by book title | `/?title=foundation` |
+| `isbn` | Filter by ISBN | `/?isbn=978-0747532699` |
+| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
+| `format` | Filter by file format | `/?format=epub` |
+| `content` | Filter by content type | `/?content=fiction` |
+| `sort` | Sort order for results | `/?sort=newest` |
+
+## Multiple Values
+
+Some parameters support multiple values by repeating the parameter:
+
+```
+/?lang=en&lang=de&lang=fr
+/?format=epub&format=mobi&format=azw3
+```
+
+## Examples
+
+**Simple search:**
+```
+/?q=lord+of+the+rings
+```
+
+**Search with author filter:**
+```
+/?q=dune&author=frank+herbert
+```
+
+**Search with format and language:**
+```
+/?q=harry+potter&format=epub&lang=en
+```
+
+**Author search with multiple formats:**
+```
+/?author=stephen+king&format=epub&format=mobi
+```
+
+**Search with sort order:**
+```
+/?q=science+fiction&sort=newest
+```
+
+## Search Mode Behavior
+
+### Direct Download Mode (default)
+
+All parameters are used to filter results from the direct download source.
+
+### Universal Mode
+
+Only `q` and `sort` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
+
+## Notes
+
+- URL parameters are read once on page load
+- The URL is not updated when you perform searches manually
+- Spaces should be encoded as `+` or `%20`
+- Invalid or unknown parameters are silently ignored

downloader.py

@@ -1,131 +0,0 @@
-"""Network operations manager for the book downloader application."""
-
-import network
-network.init()
-import requests
-import time
-from io import BytesIO
-from typing import Optional
-from urllib.parse import urlparse
-from tqdm import tqdm
-
-from logger import setup_logger
-from config import PROXIES
-from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS
-if USE_CF_BYPASS:
-    import cloudflare_bypasser
-
-logger = setup_logger(__name__)
-
-
-def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False) -> str:
-    """Fetch HTML content from a URL with retry mechanism.
-    
-    Args:
-        url: Target URL
-        retry: Number of retry attempts
-        skip_404: Whether to skip 404 errors
-        
-    Returns:
-        str: HTML content if successful, None otherwise
-    """
-    response = None
-    try:
-        logger.debug(f"html_get_page: {url}, retry: {retry}, use_bypasser: {use_bypasser}")
-        if use_bypasser and USE_CF_BYPASS:
-            logger.info(f"GET Using Cloudflare Bypasser for: {url}")
-            response_html = cloudflare_bypasser.get(url)
-            logger.debug(f"Cloudflare Bypasser response length: {len(response_html)}")
-            if response_html.strip() != "":
-                return response_html
-            else:
-                raise requests.exceptions.RequestException("Failed to bypass Cloudflare")
-        else:
-            logger.info(f"GET: {url}")
-            response = requests.get(url, proxies=PROXIES)
-            response.raise_for_status()
-            logger.debug(f"Success getting: {url}")
-            time.sleep(1)
-        return str(response.text)
-        
-    except Exception as e:
-        if retry == 0:
-            logger.error_trace(f"Failed to fetch page: {url}, error: {e}")
-            return ""
-        
-        if use_bypasser and USE_CF_BYPASS:
-            logger.warning(f"Exception while using cloudflare bypass for URL: {url}")
-            logger.warning(f"Exception: {e}")
-            logger.warning(f"Response: {response}")
-        elif response is not None and response.status_code == 404:
-            logger.warning(f"404 error for URL: {url}")
-            return ""
-        elif response is not None and response.status_code == 403:
-            logger.warning(f"403 detected for URL: {url}. Should retry using cloudflare bypass.")
-            return html_get_page(url, retry - 1, True)
-            
-        sleep_time = DEFAULT_SLEEP * (MAX_RETRY - retry + 1)
-        logger.warning(
-            f"Retrying GET {url} in {sleep_time} seconds due to error: {e}"
-        )
-        time.sleep(sleep_time)
-        return html_get_page(url, retry - 1, use_bypasser)
-
-def download_url(link: str, size: str = "") -> Optional[BytesIO]:
-    """Download content from URL into a BytesIO buffer.
-    
-    Args:
-        link: URL to download from
-        
-    Returns:
-        BytesIO: Buffer containing downloaded content if successful
-    """
-    try:
-        logger.info(f"Downloading from: {link}")
-        response = requests.get(link, stream=True, proxies=PROXIES)
-        response.raise_for_status()
-
-        total_size : float = 0.0
-        try:
-            # we assume size is in MB
-            total_size = float(size.strip().replace(" ", "").replace(",", ".").upper()[:-2].strip()) * 1024 * 1024
-        except:
-            total_size = float(response.headers.get('content-length', 0))
-        
-        buffer = BytesIO()
-
-        # Initialize the progress bar with your guess
-        pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
-        for chunk in response.iter_content(chunk_size=1000):
-            buffer.write(chunk)
-            pbar.update(len(chunk))
-            
-        pbar.close()
-        if buffer.tell() * 0.1 < total_size * 0.9:
-            # Check the content of the buffer if its HTML or binary
-            if response.headers.get('content-type', '').startswith('text/html'):
-                logger.warn(f"Failed to download content for {link}. Found HTML content instead.")
-                return None
-        return buffer
-    except requests.exceptions.RequestException as e:
-        logger.error_trace(f"Failed to download from {link}: {e}")
-        return None
-
-def get_absolute_url(base_url: str, url: str) -> str:
-    """Get absolute URL from relative URL and base URL.
-    
-    Args:
-        base_url: Base URL
-        url: Relative URL
-    """
-    if url.strip() == "":
-        return ""
-    if url.strip("#") == "":
-        return ""
-    if url.startswith("http"):
-        return url
-    parsed_url = urlparse(url)
-    parsed_base = urlparse(base_url)
-    if parsed_url.netloc == "" or parsed_url.scheme == "":
-        parsed_url = parsed_url._replace(netloc=parsed_base.netloc, scheme=parsed_base.scheme)
-    return parsed_url.geturl()

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/cwa-book-downloader
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/cwa-bd_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+
+    # Cleanup any existing files or folders in the log directory
+    rm -rf "$LOG_DIR"/*
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,14 +56,21 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
 echo "Build version: $BUILD_VERSION"
+echo "Release version: $RELEASE_VERSION"
 
 # Configure timezone
 if [ "$TZ" ]; then
@@ -27,34 +78,57 @@ if [ "$TZ" ]; then
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 fi
 
-# Set UID if not set
-if [ -z "$UID" ]; then
-    UID=1000
+# Determine user ID with proper precedence:
+# 1. PUID (LinuxServer.io standard - recommended)
+# 2. UID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+#
+# Note: $UID is a bash builtin that's always set. We use `printenv` to detect
+# if UID was explicitly set as an environment variable (e.g., via docker-compose).
+if [ -n "$PUID" ]; then
+    RUN_UID="$PUID"
+    echo "Using PUID=$RUN_UID"
+elif printenv UID >/dev/null 2>&1; then
+    RUN_UID="$(printenv UID)"
+    echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)"
+else
+    RUN_UID=1000
+    echo "Using default UID=$RUN_UID"
 fi
 
-# Set GID if not set
-if [ -z "$GID" ]; then
-    GID=100
+# Determine group ID with proper precedence:
+# 1. PGID (LinuxServer.io standard - recommended)
+# 2. GID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+if [ -n "$PGID" ]; then
+    RUN_GID="$PGID"
+    echo "Using PGID=$RUN_GID"
+elif [ -n "$GID" ]; then
+    RUN_GID="$GID"
+    echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)"
+else
+    RUN_GID=1000
+    echo "Using default GID=$RUN_GID"
 fi
 
-if ! getent group "$GID" >/dev/null; then
-    echo "Adding group $GID with name appuser"
-    groupadd -g "$GID" appuser
+if ! getent group "$RUN_GID" >/dev/null; then
+    echo "Adding group $RUN_GID with name appuser"
+    groupadd -g "$RUN_GID" appuser
 fi
 
 # Create user if it doesn't exist
-if ! id -u "$UID" >/dev/null 2>&1; then
-    echo "Adding user $UID with name appuser"
-    useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser
+if ! id -u "$RUN_UID" >/dev/null 2>&1; then
+    echo "Adding user $RUN_UID with name appuser"
+    useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser
 fi
 
 # Get username for the UID (whether we just created it or it existed)
-USERNAME=$(getent passwd "$UID" | cut -d: -f1)
-echo "Username for UID $UID is $USERNAME"
+USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1)
+echo "Username for UID $RUN_UID is $USERNAME"
 
 test_write() {
     folder=$1
-    test_file=$folder/calibre-web-automated-book-downloader_TEST_WRITE
+    test_file=$folder/shelfmark_TEST_WRITE
     mkdir -p $folder
     (
         echo 0123456789_TEST | sudo -E -u "$USERNAME" HOME=/app tee $test_file > /dev/null
@@ -83,7 +157,16 @@ make_writable() {
     else
         echo "Folder $folder is not writable, changing ownership"
         change_ownership $folder
-        chmod g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        chmod -R g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+    fi
+    # Fix any misowned subdirectories/files (e.g., from previous runs as root)
+    if [ -d "$folder" ]; then
+        misowned_count=$(find "$folder" -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) 2>/dev/null | wc -l)
+        if [ "$misowned_count" -gt 0 ]; then
+            echo "Fixing ownership of $misowned_count files/directories in $folder"
+            find "$folder" -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+                -exec chown "$RUN_UID:$RUN_GID" {} \; 2>/dev/null || true
+        fi
     fi
     test_write $folder || echo "Failed to test write to ${folder}, continuing..."
 }
@@ -92,28 +175,91 @@ make_writable() {
 change_ownership() {
   folder=$1
   mkdir -p $folder
-  echo "Changing ownership of $folder to $USERNAME:$GID"
-  chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
-  chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
+  echo "Changing ownership of $folder to $USERNAME:$RUN_GID"
+  chown -R "${RUN_UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
+  chown -R ":${RUN_GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
 }
 
 change_ownership /app
-change_ownership /var/log/cwa-book-downloader
-change_ownership /tmp/cwa-book-downloader
+change_ownership /var/log/shelfmark
+change_ownership /tmp/shelfmark
 
-# Test write to all folders
-make_writable /cwa-book-ingest
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
 
-# Set the command to run based on the environment
-is_prod=$(echo "$APP_ENV" | tr '[:upper:]' '[:lower:]')
-if [ "$is_prod" = "prod" ]; then 
-    command="gunicorn -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} app:app"
-else
-    command="python3 app.py"
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
 fi
 
-# IF DEBUG
-if [ "$DEBUG" = "true" ]; then
+# Test write to all folders
+make_writable ${CONFIG_DIR:-/config}
+make_writable ${INGEST_DIR:-/books}
+
+# Fix permissions on directories configured in settings
+echo "Checking for additional configured directories..."
+if [ -f /app/scripts/fix_permissions.py ]; then
+    configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
+    if [ -n "$configured_dirs" ]; then
+        echo "$configured_dirs" | while read -r dir; do
+            if [ -n "$dir" ] && [ -d "$dir" ]; then
+                echo "Checking configured directory: $dir"
+                make_writable "$dir"
+            fi
+        done
+    fi
+fi
+
+# Fallback to root if config dir is still not writable (common on NAS/Unraid after upgrade from v0.4.0)
+CONFIG_PATH=${CONFIG_DIR:-/config}
+set +e
+test_write "$CONFIG_PATH" >/dev/null 2>&1
+config_ok=$?
+set -e
+
+if [ $config_ok -ne 0 ] && [ "$RUN_UID" != "0" ]; then
+    config_owner=$(stat -c '%u' "$CONFIG_PATH" 2>/dev/null || echo "unknown")
+    if [ "$config_owner" = "0" ]; then
+        echo ""
+        echo "========================================================"
+        echo "WARNING: Permission issue detected!"
+        echo ""
+        echo "Config directory is owned by root but PUID=$RUN_UID."
+        echo "This typically happens after upgrading from v0.4.0 where"
+        echo "PUID/PGID settings were not respected."
+        echo ""
+        echo "Falling back to running as root to prevent data loss."
+        echo ""
+        echo "To fix this permanently, run on your HOST machine:"
+        echo "  chown -R $RUN_UID:$RUN_GID /path/to/config"
+        echo ""
+        echo "Then restart the container."
+        echo "========================================================"
+        echo ""
+        RUN_UID=0
+        RUN_GID=0
+        USERNAME=root
+    fi
+fi
+
+# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
+# upgrades work reliably on customer machines.
+# Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
+gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
+command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
+
+# If DEBUG and not using an external bypass
+if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     set +e
     set -x
     echo "vvvvvvvvvvvv DEBUG MODE vvvvvvvvvvvv"
@@ -166,18 +312,24 @@ if [ "$DEBUG" = "true" ]; then
     echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
 fi
 
-# Hacky way to verify /tmp has at least 1MB of space and is writable/readable
+# Verify /tmp has at least 1MB of space and is writable/readable
 echo "Verifying /tmp has enough space"
-rm -f /tmp/test.cwa-bd
-for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd
-sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))")
-[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1)
-rm /tmp/test.cwa-bd
+rm -f /tmp/test.shelfmark
+if dd if=/dev/zero of=/tmp/test.shelfmark bs=1M count=1 2>/dev/null && \
+   [ "$(wc -c < /tmp/test.shelfmark)" -eq 1048576 ]; then
+    rm -f /tmp/test.shelfmark
+    echo "Success: /tmp is writable and readable"
+else
+    echo "Failure: /tmp is not writable or has insufficient space"
+    exit 1
+fi
 
-echo "Running command: '$command' as '$USERNAME' in '$APP_ENV' mode"
+echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
+# Set umask for file permissions (default: 0022 = files 644, dirs 755)
+UMASK_VALUE=${UMASK:-0022}
+echo "Setting umask to $UMASK_VALUE"
+umask $UMASK_VALUE
 
+stop_file_logging
 exec sudo -E -u "$USERNAME" HOME=/app $command

env.py

@@ -1,55 +0,0 @@
-import os
-from pathlib import Path
-
-def string_to_bool(s: str) -> bool:
-    return s.lower() in ["true", "yes", "1", "y"]
-
-CWA_DB = os.getenv("CWA_DB_PATH")
-CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
-LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
-LOG_DIR = LOG_ROOT / "cwa-book-downloader"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
-INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
-STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
-USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
-MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
-DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
-USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
-HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
-HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
-AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
-_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
-_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
-_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
-_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
-_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
-FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
-FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
-PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
-
-# If debug is true, we want to log everything
-if DEBUG:
-    LOG_LEVEL = "DEBUG"
-else:
-    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
-MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
-MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "5"))
-DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip()
-USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
-BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
-APP_ENV = os.getenv("APP_ENV", "prod").lower()
-# Logging settings
-LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
-
-USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
-# If using Tor, we don't need to set custom DNS, use DOH, or proxy
-if USING_TOR:
-    _CUSTOM_DNS = ""
-    USE_DOH = False
-    HTTP_PROXY = ""
-    HTTPS_PROXY = ""
-    

genDebug.sh

@@ -2,8 +2,8 @@
 
 # Set up log paths
 LOG_ROOT=${LOG_ROOT:-"/var/log"}
-LOG_DIR="$LOG_ROOT/cwa-book-downloader"
-OUTPUT_FILE_NAME="cwa-book-downloader-debug_BUILD-${BUILD_VERSION:-local}_$(date +%Y%m%d-%H%M%S)"
+LOG_DIR="$LOG_ROOT/shelfmark"
+OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
 OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
 
 # Create LOG_DIR if it doesn't exist
@@ -18,17 +18,17 @@ echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add disk usage
 echo "=== Disk Usage ===" >> "$LOG_DIR/system_info.txt"
-df -h >> "$LOG_DIR/system_info.txt"
+df -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add memory info
 echo "=== Memory Info ===" >> "$LOG_DIR/system_info.txt"
-free -h >> "$LOG_DIR/system_info.txt"
+free -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add running processes
 echo "=== Running Processes ===" >> "$LOG_DIR/system_info.txt"
-ps aux >> "$LOG_DIR/system_info.txt"
+ps aux >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add network information using basic commands
@@ -37,17 +37,17 @@ echo "=== Network Information ===" > "$LOG_DIR/network_info.txt"
 # Try to get basic connectivity information
 echo "=== Basic Connectivity ===" >> "$LOG_DIR/network_info.txt"
 echo "Hostname resolution:" >> "$LOG_DIR/network_info.txt"
-cat /etc/hosts 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
+cat /etc/hosts >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 echo "DNS configuration:" >> "$LOG_DIR/network_info.txt"
-cat /etc/resolv.conf 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
+cat /etc/resolv.conf >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try to get interface information from /proc
 echo "=== Network Interfaces (/proc) ===" >> "$LOG_DIR/network_info.txt"
 if [ -f "/proc/net/dev" ]; then
-  cat /proc/net/dev >> "$LOG_DIR/network_info.txt"
+  cat /proc/net/dev >> "$LOG_DIR/network_info.txt" 2>&1
 else
   echo "Not available: /proc/net/dev not found" >> "$LOG_DIR/network_info.txt"
 fi
@@ -55,9 +55,9 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try connectivity tests
 echo "=== Internet Connectivity ===" >> "$LOG_DIR/network_info.txt"
-ping -c 3 1.1.1.1 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
+ping -c 3 1.1.1.1 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
-ping -c 3 one.one.one.one 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
+ping -c 3 one.one.one.one >> "$LOG_DIR/network_info.txt" 2>&1 || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test IPv6 connectivity
@@ -77,7 +77,7 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try IPv6 connectivity test using Cloudflare's IPv6 DNS
 echo "Testing IPv6 connectivity to Cloudflare DNS:" >> "$LOG_DIR/network_info.txt"
-ping6 -c 3 2606:4700:4700::1111 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
+ping6 -c 3 2606:4700:4700::1111 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test SSL connectivity
@@ -92,24 +92,36 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Add installed packages
 echo "=== Installed Python Packages ===" > "$LOG_DIR/packages.txt"
-pip list 2>/dev/null >> "$LOG_DIR/packages.txt" || echo "pip not found" >> "$LOG_DIR/packages.txt"
+pip list >> "$LOG_DIR/packages.txt" 2>&1 || echo "pip not found" >> "$LOG_DIR/packages.txt"
 echo "" >> "$LOG_DIR/packages.txt"
 
 # Check Permissions
 echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
-ls -all /app >> "$LOG_DIR/permissions.txt"
+ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt"
+echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
+ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 
+# Check Iptables (NAT)
+echo "=== IPtables NAT Rules ===" > "$LOG_DIR/iptables_nat.txt"
+iptables -t nat -L -v -n >> "$LOG_DIR/iptables_nat.txt" 2>&1
+
+# Check DNS Resolution details
+echo "=== DNS Resolution Test ===" > "$LOG_DIR/dns_test.txt"
+echo "Resolving google.com:" >> "$LOG_DIR/dns_test.txt"
+nslookup google.com >> "$LOG_DIR/dns_test.txt" 2>&1
+echo "" >> "$LOG_DIR/dns_test.txt"
+echo "Resolving check.torproject.org:" >> "$LOG_DIR/dns_test.txt"
+nslookup check.torproject.org >> "$LOG_DIR/dns_test.txt" 2>&1
+
 
 # Check if running in Docker
 echo "=== Container Info ===" > "$LOG_DIR/container_info.txt"
@@ -122,19 +134,58 @@ else
 fi
 
 # Add environment variables (redacting sensitive info)
-env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt"
+env | grep -v -E "(AA_DONATOR_KEY|HARDCOVER_API_KEY|_KEY=|_SECRET=|_PASSWORD=|_TOKEN=)" | sort > "$LOG_DIR/environment.txt"
 
-echo "--- HTTPBin ---" > $LOG_DIR/network_info.txt
-pyrequests https://httpbin.org/get >> $LOG_DIR/network_info.txt
-ehco ""
+# Add configuration files (redacting sensitive values)
+CONFIG_DIR=${CONFIG_DIR:-"/config"}
+if [ -d "$CONFIG_DIR" ]; then
+  mkdir -p "$LOG_DIR/config"
+
+  # Copy and redact main settings file
+  if [ -f "$CONFIG_DIR/settings.json" ]; then
+    # Redact sensitive fields (API keys, passwords, tokens)
+    sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+      "$CONFIG_DIR/settings.json" > "$LOG_DIR/config/settings.json" 2>/dev/null
+  fi
+
+  # Copy and redact plugin config files
+  if [ -d "$CONFIG_DIR/plugins" ]; then
+    mkdir -p "$LOG_DIR/config/plugins"
+    for config_file in "$CONFIG_DIR/plugins"/*.json; do
+      if [ -f "$config_file" ]; then
+        filename=$(basename "$config_file")
+        sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+          "$config_file" > "$LOG_DIR/config/plugins/$filename" 2>/dev/null
+      fi
+    done
+  fi
+
+  echo "Configuration files copied (sensitive values redacted)" >> "$LOG_DIR/container_info.txt"
+else
+  echo "Config directory not found at $CONFIG_DIR" >> "$LOG_DIR/container_info.txt"
+fi
+
+echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
+curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
-pyrequests https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt
-ehco ""
+curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
-pyrequests https://ipinfo.io >> $LOG_DIR/network_info.txt
-ehco ""
+curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
-pyrequests https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt
+curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt 2>&1
+
+# Copy Tor logs if they exist
+if [ -f "/var/log/tor/notices.log" ]; then
+  cp "/var/log/tor/notices.log" "$LOG_DIR/tor_notices.log"
+fi
+
+# Copy Supervisor logs if they exist
+if [ -d "/var/log/supervisor" ]; then
+  cp -rf "/var/log/supervisor/" "$LOG_DIR/supervisor/"
+fi
 
 # Create the zip file directly from LOG_DIR
 ln -s "$LOG_DIR" /tmp/$OUTPUT_FILE_NAME

models.py

@@ -1,348 +0,0 @@
-"""Data structures and models used across the application."""
-
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Tuple
-from enum import Enum
-from datetime import datetime, timedelta
-from threading import Lock, Event
-from pathlib import Path
-import queue
-import time
-from env import INGEST_DIR, STATUS_TIMEOUT
-
-class QueueStatus(str, Enum):
-    """Enum for possible book queue statuses."""
-    QUEUED = "queued"
-    DOWNLOADING = "downloading"
-    AVAILABLE = "available"
-    ERROR = "error"
-    DONE = "done"
-    CANCELLED = "cancelled"
-
-@dataclass
-class QueueItem:
-    """Queue item with priority and metadata."""
-    book_id: str
-    priority: int
-    added_time: float
-    
-    def __lt__(self, other):
-        """Compare items for priority queue (lower priority number = higher precedence)."""
-        if self.priority != other.priority:
-            return self.priority < other.priority
-        return self.added_time < other.added_time
-
-@dataclass
-class BookInfo:
-    """Data class representing book information."""
-    id: str
-    title: str
-    preview: Optional[str] = None
-    author: Optional[str] = None
-    publisher: Optional[str] = None
-    year: Optional[str] = None
-    language: Optional[str] = None
-    format: Optional[str] = None
-    size: Optional[str] = None
-    info: Optional[Dict[str, List[str]]] = None
-    download_urls: List[str] = field(default_factory=list)
-    download_path: Optional[str] = None
-    priority: int = 0
-    progress: Optional[float] = None
-
-class BookQueue:
-    """Thread-safe book queue manager with priority support and cancellation."""
-    def __init__(self) -> None:
-        self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
-        self._lock = Lock()
-        self._status: dict[str, QueueStatus] = {}
-        self._book_data: dict[str, BookInfo] = {}
-        self._status_timestamps: dict[str, datetime] = {}  # Track when each status was last updated
-        self._status_timeout = timedelta(seconds=STATUS_TIMEOUT)  # 1 hour timeout
-        self._cancel_flags: dict[str, Event] = {}  # Cancellation flags for active downloads
-        self._active_downloads: dict[str, bool] = {}  # Track currently downloading books
-    
-    def add(self, book_id: str, book_data: BookInfo, priority: int = 0) -> None:
-        """Add a book to the queue with specified priority.
-        
-        Args:
-            book_id: Unique identifier for the book
-            book_data: Book information
-            priority: Priority level (lower number = higher priority)
-        """
-        with self._lock:
-            # Don't add if already exists and not in error/done state
-            if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                return
-                
-            book_data.priority = priority
-            queue_item = QueueItem(book_id, priority, time.time())
-            self._queue.put(queue_item)
-            self._book_data[book_id] = book_data
-            self._update_status(book_id, QueueStatus.QUEUED)
-    
-    def get_next(self) -> Optional[Tuple[str, Event]]:
-        """Get next book ID from queue with cancellation flag.
-        
-        Returns:
-            Tuple of (book_id, cancel_flag) or None if queue is empty
-        """
-        try:
-            queue_item = self._queue.get_nowait()
-            book_id = queue_item.book_id
-            
-            with self._lock:
-                # Check if book was cancelled while in queue
-                if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED:
-                    return self.get_next()  # Recursively get next non-cancelled item
-                
-                # Create cancellation flag for this download
-                cancel_flag = Event()
-                self._cancel_flags[book_id] = cancel_flag
-                self._active_downloads[book_id] = True
-                
-            return book_id, cancel_flag
-        except queue.Empty:
-            return None
-            
-    def _update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Internal method to update status and timestamp."""
-        self._status[book_id] = status
-        self._status_timestamps[book_id] = datetime.now()
-            
-    def update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Update status of a book in the queue."""
-        with self._lock:
-            self._update_status(book_id, status)
-            
-            # Clean up active download tracking when finished
-            if status in [QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                self._active_downloads.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-    
-    def update_download_path(self, book_id: str, download_path: str) -> None:
-        """Update the download path of a book in the queue."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].download_path = download_path
-                
-    def update_progress(self, book_id: str, progress: float) -> None:
-        """Update download progress for a book."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].progress = progress
-            
-    def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]:
-        """Get current queue status."""
-        self.refresh()
-        with self._lock:
-            result: Dict[QueueStatus, Dict[str, BookInfo]] = {status: {} for status in QueueStatus}
-            for book_id, status in self._status.items():
-                if book_id in self._book_data:
-                    result[status][book_id] = self._book_data[book_id]
-            return result
-            
-    def get_queue_order(self) -> List[Dict[str, any]]:
-        """Get current queue order for display."""
-        with self._lock:
-            queue_items = []
-            
-            # Get items from priority queue without removing them
-            temp_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    temp_items.append(item)
-                    if item.book_id in self._book_data:
-                        book_info = self._book_data[item.book_id]
-                        queue_items.append({
-                            'id': item.book_id,
-                            'title': book_info.title,
-                            'author': book_info.author,
-                            'priority': item.priority,
-                            'added_time': item.added_time,
-                            'status': self._status.get(item.book_id, QueueStatus.QUEUED)
-                        })
-                except queue.Empty:
-                    break
-            
-            # Put items back in queue
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
-            
-    def cancel_download(self, book_id: str) -> bool:
-        """Cancel a download and mark it as cancelled.
-        
-        Args:
-            book_id: Book identifier to cancel
-            
-        Returns:
-            bool: True if cancellation was successful
-        """
-        with self._lock:
-            current_status = self._status.get(book_id)
-            
-            if current_status == QueueStatus.DOWNLOADING:
-                # Signal active download to stop
-                if book_id in self._cancel_flags:
-                    self._cancel_flags[book_id].set()
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            elif current_status == QueueStatus.QUEUED:
-                # Remove from queue and mark as cancelled
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            
-            return False
-            
-    def set_priority(self, book_id: str, new_priority: int) -> bool:
-        """Change the priority of a queued book.
-        
-        Args:
-            book_id: Book identifier
-            new_priority: New priority level (lower = higher priority)
-            
-        Returns:
-            bool: True if priority was successfully changed
-        """
-        with self._lock:
-            if book_id not in self._status or self._status[book_id] != QueueStatus.QUEUED:
-                return False
-                
-            # Remove book from queue and re-add with new priority
-            temp_items = []
-            found = False
-            
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    if item.book_id == book_id:
-                        # Create new item with updated priority
-                        new_item = QueueItem(book_id, new_priority, item.added_time)
-                        temp_items.append(new_item)
-                        found = True
-                        # Update book data priority
-                        if book_id in self._book_data:
-                            self._book_data[book_id].priority = new_priority
-                    else:
-                        temp_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return found
-            
-    def reorder_queue(self, book_priorities: Dict[str, int]) -> bool:
-        """Bulk reorder queue by setting new priorities.
-        
-        Args:
-            book_priorities: Dict mapping book_id to new priority
-            
-        Returns:
-            bool: True if reordering was successful
-        """
-        with self._lock:
-            # Extract all items from queue
-            all_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    # Update priority if specified
-                    if item.book_id in book_priorities:
-                        new_priority = book_priorities[item.book_id]
-                        item = QueueItem(item.book_id, new_priority, item.added_time)
-                        # Update book data priority
-                        if item.book_id in self._book_data:
-                            self._book_data[item.book_id].priority = new_priority
-                    all_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back with updated priorities
-            for item in all_items:
-                self._queue.put(item)
-                
-            return True
-            
-    def get_active_downloads(self) -> List[str]:
-        """Get list of currently active download book IDs."""
-        with self._lock:
-            return list(self._active_downloads.keys())
-            
-    def clear_completed(self) -> int:
-        """Remove all completed, errored, or cancelled books from tracking.
-        
-        Returns:
-            int: Number of books removed
-        """
-        with self._lock:
-            to_remove = []
-            for book_id, status in self._status.items():
-                if status in [QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
-                    to_remove.append(book_id)
-            
-            removed_count = len(to_remove)
-            for book_id in to_remove:
-                self._status.pop(book_id, None)
-                self._status_timestamps.pop(book_id, None)
-                self._book_data.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-                self._active_downloads.pop(book_id, None)
-                
-            return removed_count
-        
-    def refresh(self) -> None:
-        """Remove any books that are done downloading or have stale status."""
-        with self._lock:
-            current_time = datetime.now()
-            
-            # Create a list of items to remove to avoid modifying dict during iteration
-            to_remove = []
-            
-            for book_id, status in self._status.items():
-                path = self._book_data[book_id].download_path
-                if path and not Path(path).exists():
-                    self._book_data[book_id].download_path = None
-                    path = None
-                
-                # Check for completed downloads
-                if status == QueueStatus.AVAILABLE:
-                    if not path:
-                        self._update_status(book_id, QueueStatus.DONE)
-                
-                # Check for stale status entries
-                last_update = self._status_timestamps.get(book_id)
-                if last_update and (current_time - last_update) > self._status_timeout:
-                    if status in [QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]:
-                        to_remove.append(book_id)
-            
-            # Remove stale entries
-            for book_id in to_remove:
-                del self._status[book_id]
-                del self._status_timestamps[book_id]
-                if book_id in self._book_data:
-                    del self._book_data[book_id]
-
-    def set_status_timeout(self, hours: int) -> None:
-        """Set the status timeout duration in hours."""
-        with self._lock:
-            self._status_timeout = timedelta(hours=hours)
-
-
-# Global instance of BookQueue
-book_queue = BookQueue()
-
-@dataclass
-class SearchFilters:
-    isbn: Optional[List[str]] = None
-    author: Optional[List[str]] = None
-    title: Optional[List[str]] = None
-    lang: Optional[List[str]] = None
-    sort: Optional[str] = None
-    content: Optional[List[str]] = None
-    format: Optional[List[str]] = None

network.py

@@ -1,350 +0,0 @@
-"""Network operations manager for the book downloader application."""
-
-import requests
-import urllib.request
-from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable
-import socket
-import dns.resolver
-from socket import AddressFamily, SocketKind
-import urllib.parse
-import ssl
-import ipaddress
-
-from logger import setup_logger
-from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER
-import config
-
-logger = setup_logger(__name__)
-
-# Common helper functions for DNS resolution
-def _decode_host(host: Union[str, bytes, None]) -> str:
-    """Convert host to string, handling bytes and None cases."""
-    if host is None:
-        return ""
-    if isinstance(host, bytes):
-        return host.decode('utf-8')
-    return str(host)
-
-def _decode_port(port: Union[str, bytes, int, None]) -> int:
-    """Convert port to integer, handling various input types."""
-    if port is None:
-        return 0
-    if isinstance(port, (str, bytes)):
-        return int(port)
-    return int(port)
-
-def _is_local_address(host_str: str) -> bool:
-    """Check if an address is local and should bypass custom DNS."""
-    """Check if an address is local or private and should bypass custom DNS."""
-    # Localhost checks
-    if (host_str == 'localhost' or 
-        host_str.startswith('127.') or 
-        host_str == '::1' or 
-        host_str == '0.0.0.0'):
-        return True
-        
-    # IPv4 private ranges (RFC 1918)
-    if (host_str.startswith('10.') or 
-        (host_str.startswith('172.') and 
-         len(host_str.split('.')) > 1 and 
-         16 <= int(host_str.split('.')[1]) <= 31) or
-        host_str.startswith('192.168.')):
-        return True
-        
-    # IPv6 private ranges
-    if (host_str.startswith('fc') or 
-        host_str.startswith('fd') or  # Unique local addresses (fc00::/7)
-        host_str.startswith('fe80:')):  # Link-local addresses (fe80::/10)
-        return True
-    
-    return False
-
-def _is_ip_address(host_str: str) -> bool:
-    """Check if a string is a valid IP address (IPv4 or IPv6)."""
-    try:
-        ipaddress.ip_address(host_str)
-        return True
-    except ValueError:
-        return False
-
-# Store the original getaddrinfo function
-original_getaddrinfo = socket.getaddrinfo
-
-class DoHResolver:
-    """DNS over HTTPS resolver implementation."""
-    def __init__(self, provider_url: str, hostname: str, ip: str):
-        """Initialize DoH resolver with specified provider."""
-        self.base_url = provider_url.lower().strip()
-        self.hostname = hostname  # Store the hostname for hostname-based skipping
-        self.ip = ip              # Store IP for direct connections
-        self.session = requests.Session()
-        
-        # Different headers based on provider
-        if 'google' in self.base_url:
-            self.session.headers.update({
-                'Accept': 'application/json',
-            })
-        else:
-            self.session.headers.update({
-                'Accept': 'application/dns-json',
-            })
-    
-    def resolve(self, hostname: str, record_type: str) -> List[str]:
-        """Resolve a hostname using DoH.
-        
-        Args:
-            hostname: The hostname to resolve
-            record_type: The DNS record type (A or AAAA)
-            
-        Returns:
-            List of resolved IP addresses
-        """
-        # Check if hostname is already an IP address, no need to resolve
-        if _is_ip_address(hostname):
-            logger.debug(f"Skipping DoH resolution for IP address: {hostname}")
-            return [hostname]
-            
-        # Check if hostname is a private IP address, and skip DoH if it is
-        if _is_local_address(hostname):
-            logger.debug(f"Skipping DoH resolution for private IP: {hostname}")
-            return [hostname]
-            
-        # Skip resolution for the DoH server itself to prevent recursion
-        if hostname == self.hostname:
-            logger.debug(f"Skipping DoH resolution for DoH server itself: {hostname}")
-            return [self.ip]
-            
-        try:
-            params = {
-                'name': hostname,
-                'type': 'AAAA' if record_type == 'AAAA' else 'A'
-            }
-            
-            response = self.session.get(
-                self.base_url,
-                params=params,
-                proxies=PROXIES,
-                timeout=5
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            if 'Answer' not in data:
-                logger.warning(f"DoH resolution failed for {hostname}: {data}")
-                return []
-            
-            # Extract IP addresses from the response    
-            answers = [answer['data'] for answer in data['Answer'] 
-                    if answer.get('type') == (28 if record_type == 'AAAA' else 1)]
-            logger.debug(f"Resolved {hostname} to {len(answers)} addresses using DoH: {answers}")
-            return answers
-            
-        except Exception as e:
-            logger.warning(f"DoH resolution failed for {hostname}: {e}")
-            return []
-
-def create_custom_resolver():
-    """Create a custom DNS resolver using the configured DNS servers."""
-    custom_resolver = dns.resolver.Resolver()
-    custom_resolver.nameservers = CUSTOM_DNS
-    return custom_resolver
-
-def resolve_with_custom_dns(resolver, hostname: str, record_type: str) -> List[str]:
-    """Resolve hostname using custom DNS resolver.
-    
-    Args:
-        resolver: The DNS resolver to use
-        hostname: The hostname to resolve
-        record_type: The DNS record type (A or AAAA)
-        
-    Returns:
-        List of resolved IP addresses
-    """
-    try:
-        answers = resolver.resolve(hostname, record_type)
-        return [str(answer) for answer in answers]
-    except Exception as e:
-        logger.debug(f"{record_type} resolution failed for {hostname}: {e}")
-        return []
-
-def create_custom_getaddrinfo(
-    resolve_ipv4: Callable[[str], List[str]],
-    resolve_ipv6: Callable[[str], List[str]],
-    skip_check: Optional[Callable[[str], bool]] = None
-):
-    """Create a custom getaddrinfo function that uses the provided resolvers.
-    
-    Args:
-        resolve_ipv4: Function to resolve IPv4 addresses
-        resolve_ipv6: Function to resolve IPv6 addresses
-        skip_check: Optional function to check if custom resolution should be skipped
-        
-    Returns:
-        A custom getaddrinfo function
-    """
-    def custom_getaddrinfo(
-        host: Union[str, bytes, None],
-        port: Union[str, bytes, int, None],
-        family: int = 0,
-        type: int = 0,
-        proto: int = 0,
-        flags: int = 0
-    ) -> Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]]:
-        host_str = _decode_host(host)
-        port_int = _decode_port(port)
-        
-        # Skip custom resolution for IP addresses, local addresses, or if skip check passes
-        if _is_ip_address(host_str) or _is_local_address(host_str) or (skip_check and skip_check(host_str)):
-            logger.debug(f"Using system DNS for IP address or local/private address: {host_str}")
-            return original_getaddrinfo(host, port, family, type, proto, flags)
-        
-        results: list[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]] = []
-        
-        try:
-            # Try IPv6 first if family allows it
-            if family == 0 or family == socket.AF_INET6:
-                logger.debug(f"Resolving IPv6 address for {host_str}")
-                ipv6_answers = resolve_ipv6(host_str)
-                for answer in ipv6_answers:
-                    results.append((socket.AF_INET6, cast(SocketKind, type), proto, '', (answer, port_int, 0, 0)))
-                if ipv6_answers:
-                    logger.debug(f"Found {len(ipv6_answers)} IPv6 addresses for {host_str}")
-            
-            # Then try IPv4
-            if family == 0 or family == socket.AF_INET:
-                logger.debug(f"Resolving IPv4 address for {host_str}")
-                ipv4_answers = resolve_ipv4(host_str)
-                for answer in ipv4_answers:
-                    results.append((socket.AF_INET, cast(SocketKind, type), proto, '', (answer, port_int)))
-                if ipv4_answers:
-                    logger.debug(f"Found {len(ipv4_answers)} IPv4 addresses for {host_str}")
-            
-            if results:
-                logger.debug(f"Resolved {host_str} to {len(results)} addresses")
-                return results
-                
-        except Exception as e:
-            logger.warning(f"Custom DNS resolution failed for {host_str}: {e}, falling back to system DNS")
-        
-        # Fall back to system DNS if custom resolution fails
-        try:
-            return original_getaddrinfo(host, port, family, type, proto, flags)
-        except Exception as e:
-            logger.error(f"System DNS resolution also failed for {host_str}: {e}")
-            # Last resort: Try to connect to the hostname directly
-            if family == 0 or family == socket.AF_INET:
-                logger.warning(f"Using direct hostname as last resort for {host_str}")
-                return [(socket.AF_INET, cast(SocketKind, type), proto, '', (host_str, port_int))]
-            else:
-                raise  # Re-raise the exception if we can't provide a last resort
-    
-    return custom_getaddrinfo
-
-def init_doh_resolver(doh_server: str = DOH_SERVER):
-    """Initialize DNS over HTTPS resolver.
-    
-    Args:
-        doh_server: The DoH server URL
-    """
-    # Pre-resolve the DoH server hostname to prevent recursion
-    url = urllib.parse.urlparse(doh_server)
-    server_hostname = url.hostname if url.hostname else ''
-    
-    # Use system DNS for DoH server to prevent circular dependencies
-    try:
-        # Temporarily restore original getaddrinfo to resolve DoH server
-        temp_getaddrinfo = socket.getaddrinfo
-        socket.getaddrinfo = original_getaddrinfo
-        
-        server_ip = socket.gethostbyname(server_hostname)
-        logger.info(f"DoH server {server_hostname} resolved to IP: {server_ip}")
-        
-        # Restore custom getaddrinfo if it was previously set
-        socket.getaddrinfo = temp_getaddrinfo
-    except Exception as e:
-        logger.error(f"Failed to resolve DoH server {server_hostname}: {e}")
-        # Fall back to a known public DNS if resolution fails
-        server_ip = "1.1.1.1"
-        logger.info(f"Using fallback IP for DoH server: {server_ip}")
-    
-    # Create DoH resolver
-    doh_resolver = DoHResolver(doh_server, server_hostname, server_ip)
-    
-    # Create resolver functions
-    def resolve_ipv4(hostname: str) -> List[str]:
-        return doh_resolver.resolve(hostname, 'A')
-    
-    def resolve_ipv6(hostname: str) -> List[str]:
-        return doh_resolver.resolve(hostname, 'AAAA')
-    
-    # Skip DoH resolution for the DoH server itself, IP addresses, and private addresses
-    def skip_doh(hostname: str) -> bool:
-        return (hostname == server_hostname or 
-                hostname == server_ip or 
-                _is_ip_address(hostname) or 
-                _is_local_address(hostname))
-    
-    # Replace socket.getaddrinfo with our DoH-enabled version
-    socket.getaddrinfo = cast(Any, create_custom_getaddrinfo(
-        resolve_ipv4, resolve_ipv6, skip_doh
-    ))
-    
-    logger.info("DoH resolver successfully configured and activated")
-    return doh_resolver
-
-def init_custom_resolver():
-    """Initialize custom DNS resolver using configured DNS servers."""
-    custom_resolver = create_custom_resolver()
-    
-    # Create resolver functions
-    def resolve_ipv4(hostname: str) -> List[str]:
-        return resolve_with_custom_dns(custom_resolver, hostname, 'A')
-    
-    def resolve_ipv6(hostname: str) -> List[str]:
-        return resolve_with_custom_dns(custom_resolver, hostname, 'AAAA')
-    
-    # Replace socket.getaddrinfo with our custom resolver
-    socket.getaddrinfo = cast(Any, create_custom_getaddrinfo(resolve_ipv4, resolve_ipv6))
-    
-    logger.info("Custom DNS resolver successfully configured and activated")
-    return custom_resolver
-
-# Initialize DNS resolvers based on configuration
-def init_dns_resolvers():
-    """Initialize DNS resolvers based on configuration."""
-    if len(CUSTOM_DNS) > 0:
-        init_custom_resolver()
-        if DOH_SERVER:
-            init_doh_resolver()
-
-# Initialize DNS resolvers
-init_dns_resolvers()
-
-# Check available AA_BASE_URLs if set to auto
-if AA_BASE_URL == "auto":
-    logger.info(f"AA_BASE_URL: auto, checking available urls {AA_AVAILABLE_URLS}")
-    for url in AA_AVAILABLE_URLS:
-        try:
-            response = requests.get(url, proxies=PROXIES)
-            if response.status_code == 200:
-                AA_BASE_URL = url
-                break
-        except Exception as e:
-            logger.error_trace(f"Error checking {url}: {e}")
-    if AA_BASE_URL == "auto":
-        AA_BASE_URL = AA_AVAILABLE_URLS[0]
-config.AA_BASE_URL = AA_BASE_URL
-logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
-
-# Configure urllib opener with appropriate headers
-opener = urllib.request.build_opener()
-opener.addheaders = [
-    ('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        'AppleWebKit/537.36 (KHTML, like Gecko) '
-        'Chrome/129.0.0.0 Safari/537.3')
-]
-urllib.request.install_opener(opener)
-
-# Need an empty function to be called by downloader.py
-def init():
-    pass

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "shelfmark"
+version = "0.1.0"
+description = "Shelfmark - Book Downloader"
+requires-python = ">=3.10"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--tb=short",
+]
+markers = [
+    "integration: marks tests that require running services (deselect with '-m \"not integration\"')",
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks end-to-end tests that require the full application stack",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+ignore_missing_imports = true

readme.md

@@ -1,256 +1,258 @@
-# 📚 Calibre-Web-Automated-Book-Downloader
+# 📚 Shelfmark: Book Downloader
 
-![Calibre-Web Automated Book Downloader](static/media/logo.png 'Calibre-Web Automated Book Downloader')
+Formerly *Calibre Web Automated Book Downloader (CWABD)*
 
-An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
+<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
+
+Shelfmark is a unified web interface for searching and aggregating books and audiobook downloads from multiple sources - all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library.
+
+**Fully standalone** - no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated), [Booklore](https://github.com/booklore-app/booklore) or [Audiobookshelf](https://github.com/advplyr/audiobookshelf) for automatic import.
 
 ## ✨ Features
 
-- 🌐 User-friendly web interface for book search and download
-- 🔄 Automated download to your specified ingest folder
-- 🔌 Seamless integration with Calibre-Web-Automated
-- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr)
-- 🛡️ Cloudflare bypass capability for reliable downloads
-- 🐳 Docker-based deployment for quick setup
+- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
+- **Multiple sources** - Popular archive websites, Torrent, Usenet and IRC download support
+- **Audiobook support** - Full audiobook search and download with dedicated processing
+- **Real-Time Progress** - Unified download queue with live status updates across all sources
+- **Two Search Modes**:
+  - **Direct** - Search popular web sources
+  - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
+- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
 
 ## 🖼️ Screenshots
 
-![Main search interface Screenshot](README_images/search.png 'Main search interface')
+**Home screen**
+![Home screen](README_images/homescreen.png 'Home screen')
 
-![Details modal Screenshot placeholder](README_images/details.png 'Details modal')
+**Search results**
+![Search results](README_images/search-results.png 'Search results')
 
-![Download queue Screenshot placeholder](README_images/downloading.png 'Download queue')
+**Multi-source downloads**
+![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
+
+**Download queue**
+![Download queue](README_images/downloads.png 'Download queue')
 
 ## 🚀 Quick Start
 
 ### Prerequisites
 
-- Docker
-- Docker Compose
-- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended)
+- Docker & Docker Compose
 
-### Installation Steps
-
-1. Get the docker-compose.yml:
+### Installation
 
+1. Download the [docker-compose file](compose/docker-compose.yml):
    ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml
+   curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
    ```
 
 2. Start the service:
-
    ```bash
    docker compose up -d
    ```
 
-3. Access the web interface at `http://localhost:8084`
+3. Open `http://localhost:8084`
 
-## ⚙️ Configuration
+That's it! Configure settings through the web interface as needed.
 
-### Environment Variables
-
-#### Application Settings
-
-| Variable          | Description             | Default Value      |
-| ----------------- | ----------------------- | ------------------ |
-| `FLASK_PORT`      | Web interface port      | `8084`             |
-| `FLASK_HOST`      | Web interface binding   | `0.0.0.0`          |
-| `DEBUG`           | Debug mode toggle       | `false`            |
-| `INGEST_DIR`      | Book download directory | `/cwa-book-ingest` |
-| `TZ`              | Container timezone      | `UTC`              |
-| `UID`             | Runtime user ID         | `1000`             |
-| `GID`             | Runtime group ID        | `100`              |
-| `CWA_DB_PATH`     | Calibre-Web's database  | None               |
-| `ENABLE_LOGGING`  | Enable log file         | `true`             |
-| `LOG_LEVEL`       | Log level to use        | `info`             |
-
-If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
-
-If logging is enabld, log folder default location is `/var/log/cwa-book-downloader`
-Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages.
-
-Note that if using TOR, the TZ will be calculated automatically based on IP.
-
-#### Download Settings
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `MAX_RETRY`            | Maximum retry attempts                                    | `3`                               |
-| `DEFAULT_SLEEP`        | Retry delay (seconds)                                     | `5`                               |
-| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds)                           | `5`                               |
-| `SUPPORTED_FORMATS`    | Supported book formats                                    | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
-| `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
-| `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
-| `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
-| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
-
-If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
-
-#### AA 
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
-
-If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
-If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.
-
-#### Network Settings
-
-| Variable               | Description                     | Default Value           |
-| ---------------------- | ------------------------------- | ----------------------- |
-| `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
-| `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
-| `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | Custom DNS IP                   | ``                      |
-| `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
-
-For proxy configuration, you can specify URLs in the following format:
-```bash
-# Basic proxy
-HTTP_PROXY=http://proxy.example.com:8080
-HTTPS_PROXY=http://proxy.example.com:8080
-
-# Proxy with authentication
-HTTP_PROXY=http://username:password@proxy.example.com:8080
-HTTPS_PROXY=http://username:password@proxy.example.com:8080
-```
-
-
-The `CUSTOM_DNS` setting supports two formats:
-
-1. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
-   - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses in the same string
-
-2. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS
-   - `quad9` - Quad9 DNS
-   - `cloudflare` - Cloudflare DNS
-   - `opendns` - OpenDNS
-
-For users experiencing ISP-level website blocks (such as Virgin Media in the UK), using alternative DNS providers like Cloudflare may help bypass these restrictions
-
-If a `CUSTOM_DNS` is specified from the preset providers, you can also set a `USE_DOH=true` to force using DNS over HTTPS,
-which might also help in certain network situations. Note that only `google`, `quad9`, `cloudflare` and `opendns` are 
-supported for now, and any other value in `CUSTOM_DNS` will make the `USE_DOH` flag ignored.
-
-Try something like this :
-```bash
-CUSTOM_DNS=cloudflare
-USE_DOH=true
-```
-
-#### Custom configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `CUSTOM_SCRIPT`        | Path to an executable script that tuns after each download  | ``                      |
-
-If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation.
-
-The script is called with the full path of the downloaded file as its argument. Important notes:
-- The script must preserve the original filename for proper processing
-- The file can be modified or even deleted if needed
-- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted)
-
-You can specify these configuration in this format :
-```
-environment:
-  - CUSTOM_SCRIPT=/scripts/process-book.sh
-
-volumes:
-  - local/scripts/custom_script.sh:/scripts/process-book.sh
-```
-
-### Volume Configuration
+### Volume Setup
 
 ```yaml
 volumes:
-  - /your/local/path:/cwa-book-ingest
-  - /cwa/config/path/app.db:/auth/app.db:ro
-```
-**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777
-
-Mount should align with your Calibre-Web-Automated ingest folder.
-
-## 🧅 Tor Variant
-
-This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions.
-
-To use the Tor variant:
-
-1.  Get the Tor-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.tor.yml up -d
-    ```
-
-**Important Considerations for Tor:**
-
-*   **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying.
-*   **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set.
-*   **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor.
-
-## 🏗️ Architecture
-
-The application consists of a single service:
-
-1. **calibre-web-automated-bookdownloader**: Main application providing web interface and download functionality
-
-## 🏥 Health Monitoring
-
-Built-in health checks monitor:
-
-- Web interface availability
-- Download service status
-- Cloudflare bypass service connection
-
-Checks run every 30 seconds with a 30-second timeout and 3 retries.
-You can enable by adding this to your compose :
-```
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD pyrequests http://localhost:8084/request/api/status || exit 1
+  - /your/config/path:/config # Config, database, and artwork cache directory
+  - /your/download/path:/books # Downloaded books
+  - /client/path:/client/path # Optional: For Torrent/Usenet downloads, match your client directory exactly. 
 ```
 
-## 📝 Logging
+> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
 
-Logs are available in:
+> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
 
-- Container: `/var/logs/cwa-book-downloader.log`
-- Docker logs: Access via `docker logs`
+## ⚙️ Configuration
 
-## 🤝 Contributing
+### Search Modes
 
-Contributions are welcome! Feel free to submit a Pull Request.
+**Direct** (default)
+- Works out of the box, no setup required
+- Searches a huge library of books directly
+- Returns downloadable releases immediately
 
-## 📄 License
+**Universal**
+- Cleaner search results via metadata providers (Hardcover is recommended)
+- Aggregates releases from multiple configured sources
+- Full Audiobook support
+- Requires manual setup (API keys, additional sources)
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+### Environment Variables
 
-## ⚠️ Important Disclaimers
+Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FLASK_PORT` | Web interface port | `8084` |
+| `INGEST_DIR` | Book download directory | `/books` |
+| `TZ` | Container timezone | `UTC` |
+| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` |
+| `SEARCH_MODE` | `direct` or `universal` | `direct` |
+| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
+
+See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
+
+Some of the additional options available in Settings:
+- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
+- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
+- **IRC** - Add details for IRC book sources and download directly from the UI
+- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
+- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
+- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
+- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
+- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
+
+## 🐳 Docker Variants
+
+### Standard
+```bash
+docker compose up -d
+```
+
+The full-featured image with built-in Cloudflare bypass.
+
+#### Enable Tor Routing
+Routes all traffic through Tor for enhanced privacy:
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
+docker compose -f docker-compose.tor.yml up -d
+```
+
+**Notes:**
+- Requires `NET_ADMIN` and `NET_RAW` capabilities
+- Timezone is auto-detected from Tor exit node
+- Custom DNS/proxy settings are ignored when Tor is active
+
+### Lite
+A smaller image without the built-in Cloudflare bypasser. Ideal for:
+
+- **External bypassers** - Already running FlareSolverr or ByParr for other services
+- **Fast downloads** - Using fast download sources
+- **Alternative sources only** - Exclusively using Prowlarr, IRC, or other sources
+- **Audiobooks** - Using Shelfmark exclusively for audiobooks
+
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
+docker compose -f docker-compose.lite.yml up -d
+```
+
+If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
+
+## 🔐 Authentication
+
+Authentication is optional but recommended for shared or exposed instances. Three authentication methods are available in Settings:
+
+**1. Single Username/Password**
+
+**2. Proxy (Forward) Authentication**
+
+Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
+
+**3. Calibre-Web Database**
+
+If you're running Calibre-Web, you can reuse its user database by mounting it:
+
+```yaml
+volumes:
+  - /path/to/calibre-web/app.db:/auth/app.db:ro
+```
+
+## Health Monitoring
+
+The application exposes a health endpoint at `/api/health` (no authentication required). Add a health check to your compose:
+
+```yaml
+healthcheck:
+  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/health"]
+  interval: 30s
+  timeout: 30s
+  retries: 3
+```
+
+## Logging
+
+Logs are available via:
+- `docker logs <container-name>`
+- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
+
+Log level is configurable via Settings or `LOG_LEVEL` environment variable.
+
+## Development
+
+```bash
+# Frontend development
+make install     # Install dependencies
+make dev         # Start Vite dev server (localhost:5173)
+make build       # Production build
+make typecheck   # TypeScript checks
+
+# Backend (Docker)
+make up          # Start backend via docker-compose.dev.yml
+make down        # Stop services
+make refresh     # Rebuild and restart
+make restart     # Restart container
+```
+
+The frontend dev server proxies to the backend on port 8084.
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      Web Interface                          │
+│                 (React + TypeScript + Vite)                 │
+├─────────────────────────────────────────────────────────────┤
+│                      Flask Backend                          │
+│                   (REST API + WebSocket)                    │
+├───────────────────┬─────────────────────┬───────────────────┤
+│ Metadata Providers│   Download Queue    │  Cloudflare       │
+│                   │   & Orchestrator    │  Bypass           │
+├───────────────────┼─────────────────────┼───────────────────┤
+│ • Hardcover       │ • Task scheduling   │ • Internal        │
+│ • Open Library    │ • Progress tracking │ • External        │
+│                   │ • Retry logic       │   (FlareSolverr)  │
+├───────────────────┴─────────────────────┴───────────────────┤
+│                     Release Sources                         │
+├─────────────────────────────────────────────────────────────┤
+│ • Direct Download (Web Sources → Mirrors → Fallbacks)       │
+├─────────────────────────────────────────────────────────────┤
+│                     Network Layer                           │
+├─────────────────────────────────────────────────────────────┤
+│ • Auto DNS rotation  • Mirror failover  • Resume support    │
+└─────────────────────────────────────────────────────────────┘
+```
+
+The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
+
+## Contributing
+
+Contributions are welcome! Please file issues or submit pull requests on GitHub.
+
+> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR!
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## ⚠️ Disclaimers
 
 ### Copyright Notice
 
-While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for:
-
+This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
 - Ensuring they have the right to download requested materials
 - Respecting copyright laws and intellectual property rights
 - Using the tool in compliance with their local regulations
 
-### Duplicate Downloads Warning
+### Library Integration
 
-Please note that the current version:
+Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
 
-- Does not check for existing files in the download directory
-- Does not verify if books already exist in your Calibre database
-- Exercise caution when requesting multiple books to avoid duplicates
-
-## 💬 Support
-
-For issues or questions, please file an issue on the GitHub repository.
+## Support
 
+For issues or questions, please [file an issue](https://github.com/calibrain/shelfmark/issues) on GitHub.

requirements-base.txt

@@ -0,0 +1,16 @@
+flask
+flask-cors
+flask-socketio
+python-socketio
+requests[socks]
+beautifulsoup4
+tqdm
+dnspython
+gunicorn
+gevent
+gevent-websocket
+psutil
+emoji
+rarfile
+qbittorrent-api
+transmission-rpc

requirements-shelfmark.txt

@@ -0,0 +1,4 @@
+pyvirtualdisplay
+pyautogui
+seleniumbase>=4.45.6
+python-xlib

requirements.txt

@@ -1,11 +0,0 @@
-flask
-requests[socks]
-beautifulsoup4
-tqdm
-pyvirtualdisplay
-dnspython
-pyautogui
-seleniumbase>=4.41.1
-gunicorn
-python-xlib
-psutil

scripts/fix_permissions.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fix permissions on all configured directories.
+
+This script is called by the entrypoint to ensure all user-configured
+directories have correct ownership. It reads directory paths from:
+- CONFIG_DIR environment variable
+- Config files in CONFIG_DIR/plugins/
+
+Outputs directory paths that need permission fixing (one per line).
+The entrypoint handles the actual chown operations.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def get_directories_from_config() -> set[str]:
+    """Extract all directory paths from config files."""
+    directories = set()
+
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    plugins_dir = config_dir / "plugins"
+
+    if not plugins_dir.exists():
+        return directories
+
+    # Keys that contain directory paths
+    directory_keys = {
+        # Main destinations
+        "DESTINATION",
+        "DESTINATION_AUDIOBOOK",
+        # Content type routing directories
+        "AA_CONTENT_TYPE_DIR_FICTION",
+        "AA_CONTENT_TYPE_DIR_NON_FICTION",
+        "AA_CONTENT_TYPE_DIR_UNKNOWN",
+        "AA_CONTENT_TYPE_DIR_MAGAZINE",
+        "AA_CONTENT_TYPE_DIR_COMIC",
+        "AA_CONTENT_TYPE_DIR_STANDARDS",
+        "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+        "AA_CONTENT_TYPE_DIR_OTHER",
+        # Legacy keys (in case of old configs)
+        "INGEST_DIR",
+        "INGEST_DIR_AUDIOBOOK",
+        "INGEST_DIR_BOOK_FICTION",
+        "INGEST_DIR_BOOK_NON_FICTION",
+        "INGEST_DIR_BOOK_UNKNOWN",
+        "INGEST_DIR_MAGAZINE",
+        "INGEST_DIR_COMIC_BOOK",
+        "INGEST_DIR_STANDARDS_DOCUMENT",
+        "INGEST_DIR_MUSICAL_SCORE",
+        "INGEST_DIR_OTHER",
+        "LIBRARY_PATH",
+        "LIBRARY_PATH_AUDIOBOOK",
+    }
+
+    # Read all JSON config files
+    for config_file in plugins_dir.glob("*.json"):
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+
+            for key in directory_keys:
+                if key in config:
+                    value = config[key]
+                    if value and isinstance(value, str) and value.startswith("/"):
+                        directories.add(value)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    return directories
+
+
+def main():
+    """Output all configured directories that exist."""
+    directories = get_directories_from_config()
+
+    # Filter to directories that actually exist
+    existing = []
+    for dir_path in directories:
+        path = Path(dir_path)
+        if path.exists() and path.is_dir():
+            existing.append(dir_path)
+
+    # Output one directory per line
+    for dir_path in sorted(existing):
+        print(dir_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_env_docs.py

@@ -0,0 +1,433 @@
+#!/usr/bin/env python3
+"""Generate markdown documentation for environment variables from the settings registry.
+
+This script extracts all settings that support environment variable configuration
+and generates a comprehensive markdown file documenting each option.
+
+Usage:
+    python scripts/generate_env_docs.py [--output path/to/output.md]
+
+The generated documentation includes:
+- Environment variable name
+- Description
+- Type (string, number, boolean, etc.)
+- Default value
+- Organizational grouping by settings tab/group
+"""
+
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Add project root to path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+
+def get_field_type_name(field) -> str:
+    """Get a human-readable type name for a field."""
+    from shelfmark.core.settings_registry import (
+        CheckboxField,
+        MultiSelectField,
+        NumberField,
+        OrderableListField,
+        PasswordField,
+        SelectField,
+        TextField,
+    )
+
+    if isinstance(field, CheckboxField):
+        return "boolean"
+    elif isinstance(field, NumberField):
+        return "number"
+    elif isinstance(field, SelectField):
+        return "string (choice)"
+    elif isinstance(field, MultiSelectField):
+        return "string (comma-separated)"
+    elif isinstance(field, OrderableListField):
+        return "JSON array"
+    elif isinstance(field, PasswordField):
+        return "string (secret)"
+    elif isinstance(field, TextField):
+        return "string"
+    else:
+        return "string"
+
+
+def format_default_value(field) -> str:
+    """Format the default value for display."""
+    default = field.default
+
+    if default is None:
+        return "_none_"
+    elif isinstance(default, bool):
+        return f"`{str(default).lower()}`"
+    elif isinstance(default, (int, float)):
+        return f"`{default}`"
+    elif isinstance(default, str):
+        if default == "":
+            return "_empty string_"
+        return f"`{default}`"
+    elif isinstance(default, list):
+        if not default:
+            return "_empty list_"
+        # For simple lists, show comma-separated values
+        if all(isinstance(item, str) for item in default):
+            return f"`{','.join(default)}`"
+        # For complex lists (e.g., OrderableListField defaults), summarize
+        return f"_see UI for defaults_"
+    else:
+        return f"`{default}`"
+
+
+def get_select_options(field) -> Optional[List[str]]:
+    """Get the available options for a SelectField.
+
+    Returns options formatted as 'value (label)' or just 'value' if they match,
+    so users know the actual values to use in environment variables.
+    """
+    from shelfmark.core.settings_registry import SelectField
+
+    if not isinstance(field, SelectField):
+        return None
+
+    options = field.options
+    if callable(options):
+        try:
+            options = options()
+        except Exception:
+            return None
+
+    if not options:
+        return None
+
+    result = []
+    for opt in options:
+        value = opt.get("value", "")
+        label = opt.get("label", "")
+
+        # Format as "value (label)" unless they're the same or value is empty
+        if value == "":
+            result.append(f'`""` ({label})')
+        elif value == label or not label:
+            result.append(f"`{value}`")
+        else:
+            result.append(f"`{value}` ({label})")
+
+    return result
+
+
+def _generate_bootstrap_env_docs() -> List[str]:
+    """Generate documentation for bootstrap environment variables from env.py."""
+    # These are environment variables defined in env.py that are used before
+    # the settings registry is available
+    bootstrap_vars = [
+        {
+            "name": "CONFIG_DIR",
+            "description": "Directory for storing configuration files and plugin settings.",
+            "type": "string (path)",
+            "default": "/config",
+        },
+        {
+            "name": "LOG_ROOT",
+            "description": "Root directory for log files.",
+            "type": "string (path)",
+            "default": "/var/log/",
+        },
+        {
+            "name": "TMP_DIR",
+            "description": "Staging directory for downloads before moving to destination.",
+            "type": "string (path)",
+            "default": "/tmp/shelfmark",
+        },
+        {
+            "name": "ENABLE_LOGGING",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
+            "type": "boolean",
+            "default": "true",
+        },
+        {
+            "name": "FLASK_HOST",
+            "description": "Host address for the Flask web server.",
+            "type": "string",
+            "default": "0.0.0.0",
+        },
+        {
+            "name": "FLASK_PORT",
+            "description": "Port number for the Flask web server.",
+            "type": "number",
+            "default": "8084",
+        },
+        {
+            "name": "SESSION_COOKIE_SECURE",
+            "description": "Enable secure cookies (requires HTTPS).",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "CWA_DB_PATH",
+            "description": "Path to the Calibre-Web database for authentication integration.",
+            "type": "string (path)",
+            "default": "/auth/app.db",
+        },
+        {
+            "name": "DOCKERMODE",
+            "description": "Indicates the application is running inside a Docker container.",
+            "type": "boolean",
+            "default": "false",
+        },
+    ]
+
+    lines = [
+        "## Bootstrap Configuration",
+        "",
+        "These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
+        "",
+        "| Variable | Description | Type | Default |",
+        "|----------|-------------|------|---------|",
+    ]
+
+    for var in bootstrap_vars:
+        lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
+
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for var in bootstrap_vars:
+        lines.append(f"#### `{var['name']}`")
+        lines.append("")
+        lines.append(var["description"])
+        lines.append("")
+        lines.append(f"- **Type:** {var['type']}")
+        lines.append(f"- **Default:** `{var['default']}`")
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def generate_env_docs() -> str:
+    """Generate markdown documentation for all environment variables."""
+    # Import settings modules to ensure all settings are registered
+    import shelfmark.config.settings  # noqa: F401
+    import shelfmark.release_sources.irc.settings  # noqa: F401
+    import shelfmark.release_sources.prowlarr.settings  # noqa: F401
+    import shelfmark.metadata_providers.hardcover  # noqa: F401
+    import shelfmark.metadata_providers.openlibrary  # noqa: F401
+    import shelfmark.metadata_providers.googlebooks  # noqa: F401
+
+    from shelfmark.core.settings_registry import (
+        ActionButton,
+        HeadingField,
+        get_all_groups,
+        get_all_settings_tabs,
+    )
+
+    tabs = get_all_settings_tabs()
+    groups = {g.name: g for g in get_all_groups()}
+
+    # Organize tabs by group
+    grouped_tabs: Dict[Optional[str], List] = {None: []}
+    for group_name in groups:
+        grouped_tabs[group_name] = []
+
+    for tab in tabs:
+        group_name = tab.group
+        if group_name not in grouped_tabs:
+            grouped_tabs[group_name] = []
+        grouped_tabs[group_name].append(tab)
+
+    # Build markdown output
+    lines = [
+        "# Environment Variables",
+        "",
+        "This document lists all configuration options that can be set via environment variables.",
+        "",
+        "> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
+        "",
+        "## Table of Contents",
+        "",
+    ]
+
+    # Generate TOC
+    toc_entries = [
+        "- [Bootstrap Configuration](#bootstrap-configuration)",
+    ]
+
+    # Ungrouped tabs first
+    for tab in grouped_tabs.get(None, []):
+        anchor = tab.display_name.lower().replace(" ", "-")
+        toc_entries.append(f"- [{tab.display_name}](#{anchor})")
+
+    # Then grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if group_tabs:
+            anchor = group.display_name.lower().replace(" ", "-")
+            toc_entries.append(f"- [{group.display_name}](#{anchor})")
+            for tab in group_tabs:
+                sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
+                toc_entries.append(f"  - [{tab.display_name}](#{sub_anchor})")
+
+    lines.extend(toc_entries)
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Add bootstrap environment variables documentation
+    lines.extend(_generate_bootstrap_env_docs())
+
+    # Generate documentation for ungrouped tabs
+    for tab in grouped_tabs.get(None, []):
+        lines.extend(_generate_tab_docs(tab))
+
+    # Generate documentation for grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if not group_tabs:
+            continue
+
+        lines.append(f"## {group.display_name}")
+        lines.append("")
+
+        for tab in group_tabs:
+            lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
+
+    return "\n".join(lines)
+
+
+def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
+    """Generate documentation for a single settings tab."""
+    from shelfmark.core.settings_registry import ActionButton, HeadingField
+
+    lines = []
+
+    # Section header
+    if group_prefix:
+        lines.append(f"### {group_prefix}: {tab.display_name}")
+        anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
+    else:
+        lines.append(f"## {tab.display_name}")
+
+    lines.append("")
+
+    # Collect env-supported fields
+    env_fields = []
+    for field in tab.fields:
+        # Skip non-value fields
+        if isinstance(field, (ActionButton, HeadingField)):
+            continue
+
+        # Skip fields that don't support ENV vars
+        if not getattr(field, "env_supported", True):
+            continue
+
+        env_fields.append(field)
+
+    if not env_fields:
+        lines.append("_No environment variables for this section._")
+        lines.append("")
+        return lines
+
+    # Generate table
+    lines.append("| Variable | Description | Type | Default |")
+    lines.append("|----------|-------------|------|---------|")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        description = field.description or field.label
+        # Clean up description for table (remove newlines, escape pipes)
+        description = description.replace("\n", " ").replace("|", "\\|").strip()
+
+        field_type = get_field_type_name(field)
+        default = format_default_value(field)
+
+        lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
+
+    lines.append("")
+
+    # Add detailed documentation for each field
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        lines.append(f"#### `{env_var}`")
+        lines.append("")
+        lines.append(f"**{field.label}**")
+        lines.append("")
+
+        if field.description:
+            lines.append(field.description)
+            lines.append("")
+
+        lines.append(f"- **Type:** {get_field_type_name(field)}")
+        lines.append(f"- **Default:** {format_default_value(field)}")
+
+        if getattr(field, "required", False):
+            lines.append("- **Required:** Yes")
+
+        if getattr(field, "requires_restart", False):
+            lines.append("- **Requires restart:** Yes")
+
+        # Show options for SelectField
+        options = get_select_options(field)
+        if options:
+            lines.append(f"- **Options:** {', '.join(options)}")
+
+        # Show constraints for NumberField
+        from shelfmark.core.settings_registry import NumberField
+        if isinstance(field, NumberField):
+            constraints = []
+            if field.min_value is not None:
+                constraints.append(f"min: {field.min_value}")
+            if field.max_value is not None:
+                constraints.append(f"max: {field.max_value}")
+            if constraints:
+                lines.append(f"- **Constraints:** {', '.join(constraints)}")
+
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate markdown documentation for environment variables"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=project_root / "docs" / "environment-variables.md",
+        help="Output file path (default: docs/environment-variables.md)",
+    )
+    parser.add_argument(
+        "--stdout",
+        action="store_true",
+        help="Print to stdout instead of file",
+    )
+    args = parser.parse_args()
+
+    docs = generate_env_docs()
+
+    if args.stdout:
+        print(docs)
+    else:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(docs)
+        print(f"Generated: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_clients.py

@@ -0,0 +1,537 @@
+#!/usr/bin/env python3
+"""
+Test script for download client implementations.
+
+Usage:
+    1. Start the test stack:
+       docker compose -f docker-compose.test-clients.yml up -d
+
+    2. Wait for containers to initialize (first run takes ~30s)
+
+    3. Run this script to verify clients are accessible:
+       python scripts/test_clients.py
+
+    4. Access cwabd at http://localhost:8084
+       - Go to Settings > Prowlarr > Download Clients
+       - Select a client from the dropdown
+       - Click "Test Connection" to verify
+
+Web UIs:
+    - cwabd:        http://localhost:8084
+    - qBittorrent:  http://localhost:8080
+    - Transmission: http://localhost:9091
+    - Deluge:       http://localhost:8112
+    - NZBGet:       http://localhost:6789
+    - SABnzbd:      http://localhost:8085
+    - rTorrent:     http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
+
+Prerequisites (for running this script locally):
+    pip install requests transmission-rpc qbittorrent-api
+
+First-Time Setup:
+    qBittorrent:
+        - Check container logs for temporary password: docker logs test-qbittorrent
+        - Login at http://localhost:8080, change password to something known
+        - Default username is 'admin'
+
+    Transmission:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    Deluge:
+        - Access Web UI at http://localhost:8112 (default password: deluge)
+
+    NZBGet:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    SABnzbd:
+        - Complete the setup wizard at http://localhost:8085
+        - API key will be auto-detected by this script
+        - In cwabd, copy API key from SABnzbd Config > General
+"""
+
+import sys
+import time
+from xmlrpc import client
+
+# Test configuration - matches docker-compose.test-clients.yml
+CONFIG = {
+    # Usenet clients
+    "nzbget": {
+        "url": "http://localhost:6789",
+        "username": "admin",
+        "password": "admin",
+    },
+    "sabnzbd": {
+        "url": "http://localhost:8085",
+        "api_key": None,  # Will be read from config on first run
+    },
+    # Torrent clients
+    "qbittorrent": {
+        "url": "http://localhost:8080",
+        "username": "admin",
+        "password": "5NCngsHXm",  # Temp password from: docker logs test-qbittorrent | grep password
+    },
+    "transmission": {
+        "url": "http://localhost:9091",
+        "username": "admin",
+        "password": "admin",
+    },
+    "deluge": {
+        "url": "http://localhost:8112",
+        "password": "deluge",
+    },
+    "rtorrent": {
+        "url": "http://localhost:8000/RPC2",
+    },
+}
+
+# Test magnet link (Ubuntu ISO - legal, small metadata)
+TEST_MAGNET = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=ubuntu-22.04.3-live-server-amd64.iso"
+
+
+def test_nzbget():
+    """Test NZBGet connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing NZBGet")
+    print("=" * 50)
+
+    url = CONFIG["nzbget"]["url"]
+    username = CONFIG["nzbget"]["username"]
+    password = CONFIG["nzbget"]["password"]
+
+    try:
+        # Test connection via JSON-RPC
+        rpc_url = f"{url}/jsonrpc"
+        response = requests.post(
+            rpc_url,
+            json={"method": "version", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("result", "unknown")
+        print(f"  Connected to NZBGet {version}")
+
+        # Test status
+        response = requests.post(
+            rpc_url,
+            json={"method": "status", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        status = response.json().get("result", {})
+        print(f"  Server state: {'Paused' if status.get('ServerPaused') else 'Running'}")
+        print(f"  Downloads in queue: {status.get('DownloadedSizeMB', 0)} MB downloaded")
+
+        print("  SUCCESS: NZBGet is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to NZBGet")
+        print("  Is the container running? docker ps | grep nzbget")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_sabnzbd():
+    """Test SABnzbd connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing SABnzbd")
+    print("=" * 50)
+
+    url = CONFIG["sabnzbd"]["url"]
+    api_key = CONFIG["sabnzbd"]["api_key"]
+
+    # Try to get API key from config if not set
+    if not api_key:
+        try:
+            import os
+            ini_path = ".local/test-clients/sabnzbd/config/sabnzbd.ini"
+            if os.path.exists(ini_path):
+                with open(ini_path) as f:
+                    for line in f:
+                        if line.startswith("api_key"):
+                            api_key = line.split("=")[1].strip()
+                            print(f"  Found API key in config: {api_key[:8]}...")
+                            break
+        except Exception as e:
+            print(f"  Could not read API key from config: {e}")
+
+    if not api_key:
+        print("  ERROR: No API key configured")
+        print("  Please access http://localhost:8085 and complete initial setup")
+        print("  Then copy the API key from Config > General")
+        return False
+
+    try:
+        # Test connection
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "version", "output": "json"},
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("version", "unknown")
+        print(f"  Connected to SABnzbd {version}")
+
+        # Test queue status
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "queue", "output": "json"},
+            timeout=10,
+        )
+        queue = response.json().get("queue", {})
+        print(f"  Queue status: {queue.get('status', 'unknown')}")
+        print(f"  Items in queue: {len(queue.get('slots', []))}")
+
+        print("  SUCCESS: SABnzbd is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to SABnzbd")
+        print("  Is the container running? docker ps | grep sabnzbd")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_qbittorrent():
+    """Test qBittorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing qBittorrent")
+    print("=" * 50)
+
+    try:
+        import qbittorrentapi
+
+        url = CONFIG["qbittorrent"]["url"]
+        username = CONFIG["qbittorrent"]["username"]
+        password = CONFIG["qbittorrent"]["password"]
+
+        # Parse URL for host/port
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+
+        client = qbittorrentapi.Client(
+            host=parsed.hostname,
+            port=parsed.port or 8080,
+            username=username,
+            password=password,
+        )
+
+        # Test connection
+        client.auth_log_in()
+        version = client.app.version
+        print(f"  Connected to qBittorrent {version}")
+
+        # Get torrent list
+        torrents = client.torrents_info()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        result = client.torrents_add(urls=TEST_MAGNET, is_paused=True)
+        if result == "Ok.":
+            # Wait a moment for it to be added
+            time.sleep(1)
+            torrents = client.torrents_info()
+            if torrents:
+                test_torrent = torrents[-1]  # Most recently added
+                print(f"  Added test torrent: {test_torrent.name[:50]}...")
+                print(f"  Status: {test_torrent.state}")
+
+                # Remove it
+                client.torrents_delete(torrent_hashes=test_torrent.hash, delete_files=True)
+                print("  Removed test torrent")
+        else:
+            print(f"  Add result: {result}")
+
+        print("  SUCCESS: qBittorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: qbittorrent-api not installed")
+        print("  Run: pip install qbittorrent-api")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Forbidden" in str(e) or "401" in str(e):
+            print("\n  Authentication failed. Check password:")
+            print("  1. docker logs test-qbittorrent | grep password")
+            print("  2. Login to http://localhost:8080 and set a known password")
+        return False
+
+
+def test_transmission():
+    """Test Transmission connection."""
+    print("\n" + "=" * 50)
+    print("Testing Transmission")
+    print("=" * 50)
+
+    try:
+        from transmission_rpc import Client
+        from urllib.parse import urlparse
+
+        url = CONFIG["transmission"]["url"]
+        parsed = urlparse(url)
+
+        client = Client(
+            host=parsed.hostname,
+            port=parsed.port or 9091,
+            username=CONFIG["transmission"]["username"],
+            password=CONFIG["transmission"]["password"],
+        )
+
+        # Test connection
+        session = client.get_session()
+        print(f"  Connected to Transmission {session.version}")
+
+        # Get torrent list
+        torrents = client.get_torrents()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent = client.add_torrent(TEST_MAGNET, paused=True)
+        print(f"  Added test torrent: {torrent.name[:50]}...")
+
+        # Get status
+        status = client.get_torrent(torrent.id)
+        print(f"  Status: {status.status} ({status.percent_done * 100:.1f}%)")
+
+        # Remove it
+        client.remove_torrent(torrent.id, delete_data=True)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: Transmission is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: transmission-rpc not installed")
+        print("  Run: pip install transmission-rpc")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_deluge():
+    """Test Deluge Web UI (JSON-RPC) connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing Deluge")
+    print("=" * 50)
+
+    base_url = CONFIG["deluge"]["url"].rstrip("/")
+    password = CONFIG["deluge"]["password"]
+    rpc_url = f"{base_url}/json"
+
+    def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
+        payload = {"id": rpc_id, "method": method, "params": list(params)}
+        resp = session.post(rpc_url, json=payload, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("error"):
+            err = data["error"]
+            if isinstance(err, dict):
+                raise Exception(err.get("message") or str(err))
+            raise Exception(str(err))
+        return data.get("result")
+
+    try:
+        session = requests.Session()
+
+        # Authenticate to Deluge Web
+        if rpc_call(session, 1, "auth.login", password) is not True:
+            raise Exception("Authentication failed (check Deluge Web UI password)")
+
+        # Ensure Deluge Web is connected to a daemon
+        if rpc_call(session, 2, "web.connected") is not True:
+            hosts = rpc_call(session, 3, "web.get_hosts") or []
+            if not hosts:
+                raise Exception(
+                    "Deluge Web UI isn't connected to Deluge core (no hosts configured). "
+                    "Add/connect a daemon in Deluge Web UI → Connection Manager."
+                )
+
+            host_id = hosts[0][0]
+            for entry in hosts:
+                if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
+                    host_id = entry[0]
+                    break
+
+            rpc_call(session, 4, "web.connect", host_id)
+
+            if rpc_call(session, 5, "web.connected") is not True:
+                raise Exception(
+                    "Deluge Web UI couldn't connect to Deluge core. "
+                    "Check Deluge Web UI → Connection Manager."
+                )
+
+        version = rpc_call(session, 6, "daemon.info")
+        print(f"  Connected to Deluge {version}")
+
+        torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
+
+        if torrent_id:
+            torrent_id = str(torrent_id)
+            print(f"  Added test torrent: {torrent_id[:20]}...")
+
+            status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
+            state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
+            progress = status.get("progress", 0) if isinstance(status, dict) else 0
+            print(f"  Status: {state} ({progress:.1f}%)")
+
+            rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
+            print("  Removed test torrent")
+        else:
+            print("  WARNING: Could not add test torrent")
+
+        print("  SUCCESS: Deluge is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to Deluge Web UI")
+        print("  Is the container running? docker ps | grep deluge")
+        return False
+    except requests.exceptions.Timeout:
+        print("  ERROR: Deluge Web UI connection timed out")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "auth" in str(e).lower() or "login" in str(e).lower():
+            print("  Check Deluge Web UI password (default: deluge)")
+        return False
+
+def test_rtorrent():
+    """Test rTorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing rTorrent")
+    print("=" * 50)
+
+    try:
+        import xmlrpc.client
+
+        url = "http://localhost:8000/RPC2"
+        client = xmlrpc.client.ServerProxy(url)
+
+        # Test connection
+        version = client.system.library_version()
+        print(f"  Connected to rTorrent {version}")
+
+        # Get torrent list
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+
+        label = "automated"
+
+        commands = []
+        if label:
+            commands.append(f"d.custom1.set={label}")
+
+        download_dir = "/downloads"
+        if download_dir:
+            commands.append(f"d.directory_base.set={download_dir}")
+
+        # rtorrent is weird in that it doesn't return the torrent ID/hash on add
+        client.load.start("", TEST_MAGNET, ";".join(commands))
+        
+        # but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
+        torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
+        print(f"  Added test torrent: {torrent_id}")
+
+        torrent_list = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal=d.hash=,cat={torrent_id}",
+            "d.hash=",
+            "d.state=",
+            "d.completed_bytes=",
+            "d.size_bytes=",
+            "d.down.rate=",
+            "d.up.rate=",
+            "d.custom1=",
+            "d.complete=",
+        )
+        torrent = torrent_list[0]
+        if not torrent:
+            print("  ERROR: Could not find added torrent in list")
+            return False
+        
+        
+        client.d.erase(torrent_id)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: rTorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: xmlrpc.client not available")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Connection refused" in str(e):
+            print("  Is the container running? docker ps | grep rtorrent")
+        return False
+
+
+def main():
+    print("Download Client Test Suite")
+    print("=" * 50)
+    print("Make sure containers are running:")
+    print("  docker compose -f docker-compose.test-clients.yml up -d")
+
+    results = {}
+
+    # Test usenet clients
+    print("\n" + "=" * 50)
+    print("USENET CLIENTS")
+    print("=" * 50)
+    results["nzbget"] = test_nzbget()
+    results["sabnzbd"] = test_sabnzbd()
+
+    # Test torrent clients
+    print("\n" + "=" * 50)
+    print("TORRENT CLIENTS")
+    print("=" * 50)
+    results["qbittorrent"] = test_qbittorrent()
+    results["transmission"] = test_transmission()
+    results["deluge"] = test_deluge()
+    results["rtorrent"] = test_rtorrent()
+
+    # Summary
+    print("\n" + "=" * 50)
+    print("SUMMARY")
+    print("=" * 50)
+
+    for client, success in results.items():
+        status = "PASS" if success else "FAIL"
+        print(f"  {client}: {status}")
+
+    passed = sum(results.values())
+    total = len(results)
+    print(f"\n  Total: {passed}/{total} passed")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

shelfmark/__init__.py

@@ -0,0 +1 @@
+"""Shelfmark - book search and download service."""

shelfmark/__main__.py

@@ -0,0 +1,8 @@
+"""Package entry point for `python -m shelfmark`."""
+
+from shelfmark.main import app, socketio
+from shelfmark.config.env import FLASK_HOST, FLASK_PORT
+from shelfmark.core.config import config
+
+if __name__ == "__main__":
+    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

shelfmark/api/__init__.py

@@ -0,0 +1 @@
+"""API module - WebSocket handling."""

shelfmark/api/websocket.py

@@ -0,0 +1,174 @@
+"""WebSocket manager for real-time status updates."""
+
+import logging
+import threading
+from typing import Optional, Dict, Any, Callable, List
+
+from flask_socketio import SocketIO
+
+logger = logging.getLogger(__name__)
+
+
+class WebSocketManager:
+    """Manages WebSocket connections and broadcasts."""
+
+    def __init__(self):
+        self.socketio: Optional[SocketIO] = None
+        self._enabled = False
+        self._connection_count = 0
+        self._connection_lock = threading.Lock()
+        self._on_first_connect_callbacks: List[Callable[[], None]] = []
+        self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
+        self._needs_rewarm = False  # Flag to trigger warmup callbacks on next connect
+
+    def init_app(self, app, socketio: SocketIO):
+        """Initialize the WebSocket manager with Flask-SocketIO instance."""
+        self.socketio = socketio
+        self._enabled = True
+        logger.info("WebSocket manager initialized")
+
+    def register_on_first_connect(self, callback: Callable[[], None]):
+        """Register a callback for when the first client connects."""
+        self._on_first_connect_callbacks.append(callback)
+        logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
+
+    def register_on_all_disconnect(self, callback: Callable[[], None]):
+        """Register a callback for when all clients disconnect."""
+        self._on_all_disconnect_callbacks.append(callback)
+        logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
+
+    def request_warmup_on_next_connect(self):
+        """Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
+        with self._connection_lock:
+            self._needs_rewarm = True
+            logger.debug("Warmup requested for next client connect")
+
+    def client_connected(self):
+        """Track a new client connection. Call this from the connect event handler."""
+        with self._connection_lock:
+            was_zero = self._connection_count == 0
+            needs_rewarm = self._needs_rewarm
+            self._connection_count += 1
+            current_count = self._connection_count
+            # Clear rewarm flag if we're going to trigger warmup
+            if was_zero or needs_rewarm:
+                self._needs_rewarm = False
+
+        logger.debug(f"Client connected. Active connections: {current_count}")
+
+        # Trigger warmup callbacks if this is the first connection OR if rewarm was requested
+        # (rewarm is requested when bypasser shuts down due to idle while clients are connected)
+        if was_zero or needs_rewarm:
+            reason = "First client connected" if was_zero else "Rewarm requested after idle shutdown"
+            logger.info(f"{reason}, triggering warmup callbacks...")
+            for callback in self._on_first_connect_callbacks:
+                try:
+                    # Run callbacks in a separate thread to not block the connection
+                    thread = threading.Thread(target=callback, daemon=True)
+                    thread.start()
+                except Exception as e:
+                    logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}")
+
+    def client_disconnected(self):
+        """Track a client disconnection. Call this from the disconnect event handler."""
+        with self._connection_lock:
+            self._connection_count = max(0, self._connection_count - 1)
+            current_count = self._connection_count
+            is_now_zero = current_count == 0
+
+        logger.debug(f"Client disconnected. Active connections: {current_count}")
+
+        # If all clients have disconnected, trigger cleanup callbacks
+        if is_now_zero:
+            logger.info("All clients disconnected, triggering disconnect callbacks...")
+            for callback in self._on_all_disconnect_callbacks:
+                try:
+                    callback()
+                except Exception as e:
+                    logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}")
+
+    def get_connection_count(self) -> int:
+        """Get the current number of active WebSocket connections."""
+        with self._connection_lock:
+            return self._connection_count
+
+    def has_active_connections(self) -> bool:
+        """Check if there are any active WebSocket connections."""
+        return self.get_connection_count() > 0
+
+    def is_enabled(self) -> bool:
+        """Check if WebSocket is enabled and ready."""
+        return self._enabled and self.socketio is not None
+
+    def broadcast_status_update(self, status_data: Dict[str, Any]):
+        """Broadcast status update to all connected clients."""
+        if not self.is_enabled():
+            return
+
+        try:
+            # When calling socketio.emit() outside event handlers, it broadcasts by default
+            self.socketio.emit('status_update', status_data)
+            logger.debug(f"Broadcasted status update to all clients")
+        except Exception as e:
+            logger.error(f"Error broadcasting status update: {e}")
+
+    def broadcast_download_progress(self, book_id: str, progress: float, status: str):
+        """Broadcast download progress update for a specific book."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'book_id': book_id,
+                'progress': progress,
+                'status': status
+            }
+            # When calling socketio.emit() outside event handlers, it broadcasts by default
+            self.socketio.emit('download_progress', data)
+            logger.debug(f"Broadcasted progress for book {book_id}: {progress}%")
+        except Exception as e:
+            logger.error(f"Error broadcasting download progress: {e}")
+
+    def broadcast_notification(self, message: str, notification_type: str = 'info'):
+        """Broadcast a notification message to all clients."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'message': message,
+                'type': notification_type
+            }
+            # When calling socketio.emit() outside event handlers, it broadcasts by default
+            self.socketio.emit('notification', data)
+            logger.debug(f"Broadcasted notification: {message}")
+        except Exception as e:
+            logger.error(f"Error broadcasting notification: {e}")
+
+    def broadcast_search_status(
+        self,
+        source: str,
+        provider: str,
+        book_id: str,
+        message: str,
+        phase: str = 'searching'
+    ):
+        """Broadcast search status update for a release source search."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'source': source,
+                'provider': provider,
+                'book_id': book_id,
+                'message': message,
+                'phase': phase,
+            }
+            self.socketio.emit('search_status', data)
+        except Exception as e:
+            logger.error(f"Error broadcasting search status: {e}")
+
+
+# Global WebSocket manager instance
+ws_manager = WebSocketManager()

shelfmark/bypass/__init__.py

@@ -0,0 +1,5 @@
+"""Cloudflare bypass utilities."""
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""

shelfmark/bypass/external_bypasser.py

@@ -0,0 +1,128 @@
+"""External Cloudflare bypasser using FlareSolverr."""
+
+import random
+import time
+from threading import Event
+from typing import TYPE_CHECKING, Optional
+
+import requests
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.utils import normalize_http_url
+
+if TYPE_CHECKING:
+    from shelfmark.download import network
+
+logger = setup_logger(__name__)
+
+# Timeout constants (seconds)
+CONNECT_TIMEOUT = 10
+MAX_READ_TIMEOUT = 120
+READ_TIMEOUT_BUFFER = 15
+
+# Retry settings
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
+
+
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service. Returns HTML or None."""
+    raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
+    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
+    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
+
+    bypasser_url = normalize_http_url(raw_bypasser_url)
+    if not bypasser_url or not bypasser_path:
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
+        return None
+
+    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+
+    try:
+        response = requests.post(
+            f"{bypasser_url}{bypasser_path}",
+            headers={"Content-Type": "application/json"},
+            json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
+            timeout=(CONNECT_TIMEOUT, read_timeout)
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+
+        solution = result.get('solution')
+        html = solution.get('response', '') if solution else ''
+
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+
+        return html
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+
+    return None
+
+
+def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
+    """Check if operation was cancelled and raise exception if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(f"External bypasser cancelled {context}")
+        raise BypassCancelledException("Bypass cancelled")
+
+
+def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
+    """Sleep for the specified duration, checking for cancellation each second."""
+    for _ in range(int(seconds)):
+        _check_cancelled(cancel_flag, "during backoff")
+        time.sleep(1)
+    remaining = seconds - int(seconds)
+    if remaining > 0:
+        time.sleep(remaining)
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional["network.AAMirrorSelector"] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML via external bypasser with retries and mirror rotation."""
+    from shelfmark.download import network as network_module
+
+    sel = selector or network_module.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        _check_cancelled(cancel_flag, "by user")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        _sleep_with_cancellation(delay, cancel_flag)
+
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

shelfmark/bypass/fingerprint.py

@@ -0,0 +1,57 @@
+"""Browser fingerprint profile management for bypass stealth."""
+
+import random
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+COMMON_RESOLUTIONS = [
+    (1920, 1080, 0.35),  
+    (1366, 768, 0.18),   
+    (1536, 864, 0.10),   
+    (1440, 900, 0.08),   
+    (1280, 720, 0.07),   
+    (1600, 900, 0.06),   
+    (1280, 800, 0.05),   
+    (2560, 1440, 0.04),  
+    (1680, 1050, 0.04),  
+    (1920, 1200, 0.03),  
+]
+
+# Current screen size (module-level singleton)
+_current_screen_size: Optional[tuple[int, int]] = None
+
+
+def get_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    if _current_screen_size is None:
+        _current_screen_size = _generate_screen_size()
+        logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
+    return _current_screen_size
+
+
+def rotate_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    old_size = _current_screen_size
+    _current_screen_size = _generate_screen_size()
+    width, height = _current_screen_size
+
+    if old_size:
+        logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
+    else:
+        logger.info(f"Generated screen size: {width}x{height}")
+
+    return _current_screen_size
+
+
+def clear_screen_size() -> None:
+    global _current_screen_size
+    _current_screen_size = None
+
+
+def _generate_screen_size() -> tuple[int, int]:
+    resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
+    weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
+    return random.choices(resolutions, weights=weights)[0]

shelfmark/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration module - environment variables and settings."""

shelfmark/config/booklore_settings.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.outputs.booklore import (
+    BookloreConfig,
+    BookloreError,
+    booklore_list_libraries,
+    booklore_login,
+)
+
+logger = setup_logger(__name__)
+
+_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
+    "key": None,
+    "library_options": [],
+    "path_options": [],
+}
+
+
+def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
+    return f"{base_url}|{username}|{hash(password)}"
+
+
+def _get_booklore_select_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    # library_id/path_id are not used for login/library listing
+    booklore_config = BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=1,
+        path_id=1,
+        verify_tls=True,
+        refresh_after_upload=True,
+    )
+
+    token = booklore_login(booklore_config)
+    libraries = booklore_list_libraries(booklore_config, token) or []
+    logger.debug("Booklore libraries response: %s", libraries)
+
+    library_options: list[dict[str, Any]] = []
+    path_options: list[dict[str, Any]] = []
+
+    for library in libraries:
+        if not isinstance(library, dict):
+            continue
+
+        library_id = library.get("id")
+        if library_id is None:
+            continue
+
+        library_name = str(library.get("name") or f"Library {library_id}")
+        library_id_str = str(library_id)
+
+        library_options.append({"value": library_id_str, "label": library_name})
+
+        paths = library.get("paths") or []
+        if not isinstance(paths, list):
+            continue
+
+        for path in paths:
+            if not isinstance(path, dict):
+                continue
+
+            path_id = path.get("id")
+            if path_id is None:
+                continue
+
+            path_label = str(path.get("path") or f"Path {path_id}")
+            path_options.append(
+                {
+                    "value": str(path_id),
+                    "label": f"{library_name}: {path_label}",
+                    "childOf": library_id_str,
+                }
+            )
+
+    logger.debug(
+        "Booklore options built: libraries=%d paths=%d",
+        len(library_options),
+        len(path_options),
+    )
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    _BOOKLORE_OPTIONS_CACHE.update(
+        {
+            "key": cache_key,
+            "library_options": library_options,
+            "path_options": path_options,
+        }
+    )
+
+    return library_options, path_options
+
+
+def _get_booklore_cached_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+        return (
+            _BOOKLORE_OPTIONS_CACHE.get("library_options", []),
+            _BOOKLORE_OPTIONS_CACHE.get("path_options", []),
+        )
+
+    return _get_booklore_select_options(base_url, username, password)
+
+
+def get_booklore_library_options() -> list[dict[str, Any]]:
+    """Build Booklore library options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        library_options, _ = _get_booklore_cached_options(base_url, username, password)
+        return library_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore libraries: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
+        return []
+
+
+def get_booklore_path_options() -> list[dict[str, Any]]:
+    """Build Booklore path options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        _, path_options = _get_booklore_cached_options(base_url, username, password)
+        return path_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore paths: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
+        return []
+
+
+def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test the Booklore connection using current form values."""
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
+    password = _get_value("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        return {"success": False, "message": "Booklore URL is required"}
+    if not username:
+        return {"success": False, "message": "Booklore username is required"}
+    if not password:
+        return {"success": False, "message": "Booklore password is required"}
+
+    try:
+        library_options, _ = _get_booklore_select_options(base_url, username, password)
+
+        message = "Connected to Booklore"
+        if library_options:
+            message = f"Connected to Booklore ({len(library_options)} libraries)"
+
+        return {"success": True, "message": message}
+    except BookloreError as exc:
+        return {"success": False, "message": str(exc)}

shelfmark/config/env.py

@@ -0,0 +1,153 @@
+"""Bootstrap environment variables. No local dependencies - import first."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+
+def string_to_bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ["true", "yes", "1", "y"]
+
+
+def _read_debug_from_config() -> bool:
+    """Read DEBUG from env var or config file (import-time safe)."""
+    env_debug = os.environ.get("DEBUG")
+    if env_debug is not None:
+        return string_to_bool(env_debug)
+
+    # Try to read from config file
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    config_file = config_dir / "plugins" / "advanced.json"
+
+    if config_file.exists():
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+                if "DEBUG" in config:
+                    return bool(config["DEBUG"])
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return False
+
+
+def _is_sqlite_file(path: Path) -> bool:
+    """Check if a file is a valid SQLite database by reading magic bytes."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(16)
+            return header[:16] == b"SQLite format 3\x00"
+    except (OSError, PermissionError):
+        return False
+
+
+def _resolve_cwa_db_path() -> Path | None:
+    """Resolve CWA database path from env var or default location."""
+    env_path = os.getenv("CWA_DB_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file() and _is_sqlite_file(path):
+            return path
+
+    # Check default mount path
+    default_path = Path("/auth/app.db")
+    if default_path.exists() and default_path.is_file() and _is_sqlite_file(default_path):
+        return default_path
+
+    return None
+
+
+def _is_config_dir_writable() -> bool:
+    """Check if the config directory exists and is writable."""
+    try:
+        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
+            return False
+        test_file = CONFIG_DIR / ".write_test"
+        test_file.touch()
+        test_file.unlink()
+        return True
+    except (OSError, PermissionError):
+        return False
+
+
+def is_covers_cache_enabled() -> bool:
+    """Check if cover caching is enabled (requires setting + writable config dir)."""
+    from shelfmark.core.config import config
+    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
+    return setting_enabled and _is_config_dir_writable()
+
+
+# =============================================================================
+# Bootstrap paths - needed before settings registry is available
+# =============================================================================
+
+CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
+LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
+LOG_DIR = LOG_ROOT / "shelfmark"
+LOG_FILE = LOG_DIR / "shelfmark.log"
+TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
+INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
+
+
+# =============================================================================
+# Logger configuration - needed before settings registry is available
+# =============================================================================
+
+DEBUG = _read_debug_from_config()
+LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
+ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+
+
+# =============================================================================
+# Flask configuration - needed before app starts
+# =============================================================================
+
+FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
+
+
+# =============================================================================
+# Authentication
+# =============================================================================
+
+SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
+CWA_DB_PATH = _resolve_cwa_db_path()
+
+
+# =============================================================================
+# Version information from Docker build
+# =============================================================================
+
+BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
+RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
+
+
+# =============================================================================
+# Capability detection - runtime checks, not user-configurable
+# =============================================================================
+
+DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
+TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
+USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
+
+
+# =============================================================================
+# Debug/development settings
+# =============================================================================
+
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
+
+
+# =============================================================================
+# Legacy migration support - will be removed in future version
+# =============================================================================
+
+# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
+# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
+_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))

shelfmark/config/security.py

@@ -0,0 +1,290 @@
+"""Authentication settings registration."""
+
+from typing import Any, Dict
+
+from werkzeug.security import generate_password_hash
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    register_settings,
+    register_on_save,
+    load_config_file,
+    TextField,
+    SelectField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+
+logger = setup_logger(__name__)
+
+
+def _migrate_security_settings() -> None:
+    import json
+    from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
+
+    try:
+        config = load_config_file("security")
+        migrated = False
+        
+        # Migrate USE_CWA_AUTH to AUTH_METHOD
+        if "USE_CWA_AUTH" in config:
+            old_value = config.pop("USE_CWA_AUTH")
+            
+            # Only set AUTH_METHOD if it doesn't already exist
+            if "AUTH_METHOD" not in config:
+                if old_value:
+                    config["AUTH_METHOD"] = "cwa"
+                    logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
+                else:
+                    # If USE_CWA_AUTH was False, determine auth method from credentials
+                    if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                        config["AUTH_METHOD"] = "builtin"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
+                    else:
+                        config["AUTH_METHOD"] = "none"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
+                migrated = True
+            else:
+                logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
+                migrated = True
+        
+        # Migrate RESTRICT_SETTINGS_TO_ADMIN to CWA_RESTRICT_SETTINGS_TO_ADMIN
+        if "RESTRICT_SETTINGS_TO_ADMIN" in config:
+            old_value = config.pop("RESTRICT_SETTINGS_TO_ADMIN")
+            
+            # Only migrate if new key doesn't exist
+            if "CWA_RESTRICT_SETTINGS_TO_ADMIN" not in config:
+                config["CWA_RESTRICT_SETTINGS_TO_ADMIN"] = old_value
+                logger.info(f"Migrated RESTRICT_SETTINGS_TO_ADMIN={old_value} to CWA_RESTRICT_SETTINGS_TO_ADMIN={old_value}")
+                migrated = True
+            else:
+                logger.info("Removed deprecated RESTRICT_SETTINGS_TO_ADMIN setting (CWA_RESTRICT_SETTINGS_TO_ADMIN already exists)")
+                migrated = True
+        
+        # Save config if any migrations occurred
+        if migrated:
+            _ensure_config_dir("security")
+            config_path = _get_config_file_path("security")
+            with open(config_path, 'w') as f:
+                json.dump(config, f, indent=2)
+            logger.info("Security settings migration completed successfully")
+        else:
+            logger.debug("No security settings migration needed")
+    
+    except FileNotFoundError:
+        logger.debug("No existing security config file found - nothing to migrate")
+    except Exception as e:
+        logger.error(f"Failed to migrate security settings: {e}")
+
+
+def _clear_builtin_credentials() -> Dict[str, Any]:
+    """Clear built-in credentials to allow public access."""
+    import json
+    from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
+
+    try:
+        config = load_config_file("security")
+        config.pop("BUILTIN_USERNAME", None)
+        config.pop("BUILTIN_PASSWORD_HASH", None)
+
+        _ensure_config_dir("security")
+        config_path = _get_config_file_path("security")
+        with open(config_path, 'w') as f:
+            json.dump(config, f, indent=2)
+
+        logger.info("Cleared credentials")
+        return {"success": True, "message": "Credentials cleared. The app is now publicly accessible."}
+
+    except Exception as e:
+        logger.error(f"Failed to clear credentials: {e}")
+        return {"success": False, "message": f"Failed to clear credentials: {str(e)}"}
+
+
+def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Custom save handler for security settings.
+
+    Handles password validation and hashing:
+    - If new password is provided, validate confirmation and hash it
+    - If password fields are empty, preserve existing hash
+    - Never store raw passwords
+    - Ensure username is present if password is set
+
+    Returns:
+        Dict with processed values to save and any validation errors.
+    """
+    password = values.get("BUILTIN_PASSWORD", "")
+    password_confirm = values.get("BUILTIN_PASSWORD_CONFIRM", "")
+
+    # Remove raw password fields - they should never be persisted
+    values.pop("BUILTIN_PASSWORD", None)
+    values.pop("BUILTIN_PASSWORD_CONFIRM", None)
+
+    # If password is provided, validate and hash it
+    if password:
+        if not values.get("BUILTIN_USERNAME"):
+            return {
+                "error": True,
+                "message": "Username cannot be empty",
+                "values": values
+            }
+
+        if password != password_confirm:
+            return {
+                "error": True,
+                "message": "Passwords do not match",
+                "values": values
+            }
+
+        if len(password) < 4:
+            return {
+                "error": True,
+                "message": "Password must be at least 4 characters",
+                "values": values
+            }
+
+        # Hash the password
+        values["BUILTIN_PASSWORD_HASH"] = generate_password_hash(password)
+        logger.info("Password hash updated")
+
+    # If no password provided but username is being set, preserve existing hash
+    elif "BUILTIN_USERNAME" in values:
+        existing = load_config_file("security")
+        if "BUILTIN_PASSWORD_HASH" in existing:
+            values["BUILTIN_PASSWORD_HASH"] = existing["BUILTIN_PASSWORD_HASH"]
+
+    return {"error": False, "values": values}
+
+
+@register_settings("security", "Security", icon="shield", order=5)
+def security_settings():  
+    """Security and authentication settings."""
+    from shelfmark.config.env import CWA_DB_PATH
+
+    cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
+
+    auth_method_options = [
+        {"label": "No Authentication", "value": "none"},
+        {"label": "Username/Password", "value": "builtin"},
+        {"label": "Proxy Authentication", "value": "proxy"},
+    ]
+    if cwa_db_available:
+        auth_method_options.append({"label": "Calibre-Web Database", "value": "cwa"})
+
+    auth_method_description = "Select the authentication method for accessing Shelfmark."
+    if not cwa_db_available:
+        auth_method_description += " Calibre-Web database option requires mounting your Calibre-Web app.db to /auth/app.db."
+
+    fields = [
+        SelectField(
+            key="AUTH_METHOD",
+            label="Authentication Method",
+            description=auth_method_description,
+            options=auth_method_options,
+            default="none",
+            env_supported=False,
+        ),
+        TextField(
+            key="BUILTIN_USERNAME",
+            label="Username",
+            description="Set a username and password to require login. Leave both empty for public access.",
+            placeholder="Enter username",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        PasswordField(
+            key="BUILTIN_PASSWORD",
+            label="Set Password",
+            description="Fill in to set or change the password.",
+            placeholder="Enter new password",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        PasswordField(
+            key="BUILTIN_PASSWORD_CONFIRM",
+            label="Confirm Password",
+            placeholder="Confirm new password",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        ActionButton(
+            key="clear_credentials",
+            label="Clear Credentials",
+            description="Remove login requirement and make the app publicly accessible.",
+            style="danger",
+            callback=_clear_builtin_credentials,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        TextField(
+            key="PROXY_AUTH_USER_HEADER",
+            label="Proxy Auth User Header",
+            description=(
+                "The HTTP header your proxy uses to pass the authenticated username."
+            ),
+            placeholder="e.g. X-Auth-User",
+            default="X-Auth-User",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        TextField(
+            key="PROXY_AUTH_LOGOUT_URL",
+            label="Proxy Auth Logout URL",
+            description=(
+                "The URL to redirect users to for logging out."
+                " Leave empty to disable logout functionality."
+            ),
+            placeholder="https://myauth.example.com/logout",
+            default="",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        CheckboxField(
+            key="PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
+            label="Restrict Settings to Admins authenticated via Proxy",
+            description=(
+                "Only users in the admin group can access settings."
+            ),
+            default=False,
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        TextField(
+            key="PROXY_AUTH_ADMIN_GROUP_HEADER",
+            label="Proxy Auth Admin Group Header",
+            description=(
+                "The HTTP header your proxy uses to pass the user's groups/roles."
+            ),
+            placeholder="e.g. X-Auth-Groups",
+            default="X-Auth-Groups",
+            env_supported=False,
+            show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
+        ),
+        TextField(
+            key="PROXY_AUTH_ADMIN_GROUP_NAME",
+            label="Proxy Auth Admin Group Name",
+            description=(
+                "The name of the group/role that should have admin access."
+            ),
+            placeholder="e.g. admins",
+            default="admins",
+            env_supported=False,
+            show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
+        ),
+        CheckboxField(
+            key="CWA_RESTRICT_SETTINGS_TO_ADMIN",
+            label="Restrict Settings to Admins authenticated via Calibre-Web",
+            description=(
+                "Only users with admin role in Calibre-Web can access settings."
+            ),
+            default=False,
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "cwa"},
+        ),
+    ]
+
+    return fields
+
+
+# Register the on_save handler for this tab
+register_on_save("security", _on_save_security)

shelfmark/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core module - shared models, queue, and utilities."""
+
+from shelfmark.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
+from shelfmark.core.queue import BookQueue, book_queue
+from shelfmark.core.logger import setup_logger

shelfmark/core/cache.py

@@ -0,0 +1,172 @@
+"""Thread-safe in-memory cache with TTL support."""
+
+import threading
+import time
+from dataclasses import dataclass
+from functools import wraps
+from typing import Any, Callable, Dict, Optional, TypeVar
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+T = TypeVar("T")
+
+
+@dataclass
+class CacheEntry:
+    """A cached value with expiration time."""
+    value: Any
+    expires_at: float
+
+
+class CacheService:
+    """Thread-safe in-memory cache with TTL support."""
+
+    def __init__(self, max_size: int = 1000):
+        """Initialize cache with max_size entries before eviction."""
+        self._cache: Dict[str, CacheEntry] = {}
+        self._lock = threading.Lock()
+        self._max_size = max_size
+
+    def get(self, key: str) -> Optional[Any]:
+        """Get cached value if not expired."""
+        with self._lock:
+            entry = self._cache.get(key)
+            if entry is None:
+                return None
+
+            if time.time() > entry.expires_at:
+                del self._cache[key]
+                return None
+
+            return entry.value
+
+    def set(self, key: str, value: Any, ttl: int) -> None:
+        """Cache value with TTL in seconds."""
+        with self._lock:
+            # Evict oldest entries if at capacity
+            if len(self._cache) >= self._max_size:
+                self._evict_oldest()
+
+            self._cache[key] = CacheEntry(
+                value=value,
+                expires_at=time.time() + ttl
+            )
+
+    def invalidate(self, key: str) -> bool:
+        """Remove specific cache entry. Returns True if found."""
+        with self._lock:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        with self._lock:
+            self._cache.clear()
+
+    def cleanup_expired(self) -> int:
+        """Remove all expired entries. Returns count removed."""
+        with self._lock:
+            now = time.time()
+            expired_keys = [
+                key for key, entry in self._cache.items()
+                if entry.expires_at < now
+            ]
+            for key in expired_keys:
+                del self._cache[key]
+            return len(expired_keys)
+
+    def _evict_oldest(self) -> None:
+        """Evict ~10% of oldest entries. Called with lock held."""
+        if not self._cache:
+            return
+
+        # Remove ~10% of entries, oldest first
+        entries_to_remove = max(1, len(self._cache) // 10)
+        sorted_entries = sorted(
+            self._cache.items(),
+            key=lambda x: x[1].expires_at
+        )
+
+        for key, _ in sorted_entries[:entries_to_remove]:
+            del self._cache[key]
+
+    def stats(self) -> Dict[str, int]:
+        """Get cache statistics (size, max_size)."""
+        with self._lock:
+            return {
+                "size": len(self._cache),
+                "max_size": self._max_size
+            }
+
+
+# Global cache instance for metadata providers
+_metadata_cache = CacheService(max_size=1000)
+
+
+def get_metadata_cache() -> CacheService:
+    """Get the global metadata cache instance."""
+    return _metadata_cache
+
+
+def cache_key(*args, **kwargs) -> str:
+    """Generate cache key from arguments."""
+    parts = [str(arg) for arg in args]
+    parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
+    return ":".join(parts)
+
+
+def cacheable(
+    ttl: Optional[int] = None,
+    ttl_key: Optional[str] = None,
+    ttl_default: int = 300,
+    key_prefix: str = ""
+):
+    """Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            # Check if metadata caching is enabled
+            from shelfmark.core.config import config
+
+            if not config.get("METADATA_CACHE_ENABLED", True):
+                # Caching disabled, execute function directly
+                return func(*args, **kwargs)
+
+            # Determine TTL: static or from config
+            if ttl is not None:
+                effective_ttl = ttl
+            elif ttl_key:
+                effective_ttl = config.get(ttl_key, ttl_default)
+            else:
+                effective_ttl = ttl_default
+
+            # Generate cache key from function name and arguments
+            # Skip 'self' argument if present (first arg of method)
+            cache_args = args[1:] if args and hasattr(args[0], func.__name__) else args
+
+            key = cache_key(
+                key_prefix or func.__name__,
+                *cache_args,
+                **kwargs
+            )
+
+            # Check cache
+            cached = _metadata_cache.get(key)
+            if cached is not None:
+                return cached
+
+            # Execute function and cache result
+            result = func(*args, **kwargs)
+
+            # Only cache non-None results
+            if result is not None:
+                _metadata_cache.set(key, result, effective_ttl)
+
+            return result
+
+        return wrapper
+    return decorator

shelfmark/core/config.py

@@ -0,0 +1,183 @@
+"""Configuration singleton with ENV > config file > default resolution."""
+
+from threading import Lock
+from typing import Any, Dict, Optional
+
+# Import lazily to avoid circular imports
+_registry_module = None
+_env_module = None
+
+
+def _get_registry():
+    """Lazy import of settings registry to avoid circular imports."""
+    global _registry_module
+    if _registry_module is None:
+        from shelfmark.core import settings_registry
+        _registry_module = settings_registry
+    return _registry_module
+
+
+def _get_env():
+    """Lazy import of env module for fallback values."""
+    global _env_module
+    if _env_module is None:
+        from shelfmark.config import env
+        _env_module = env
+    return _env_module
+
+
+class Config:
+    """
+    Dynamic configuration singleton that provides live settings access.
+
+    Settings are resolved with priority: ENV var > config file > default.
+    Values are cached for performance and can be refreshed when settings change.
+    """
+
+    _instance: Optional['Config'] = None
+    _lock = Lock()
+
+    def __new__(cls) -> 'Config':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+        self._cache: Dict[str, Any] = {}
+        self._field_map: Dict[str, tuple] = {}  # key -> (field, tab_name)
+        self._cache_lock = Lock()
+        self._initialized = True
+        self._loaded = False
+
+    def _ensure_loaded(self) -> None:
+        """Ensure settings are loaded from the registry."""
+        if self._loaded:
+            return
+        with self._cache_lock:
+            if self._loaded:
+                return
+            self._load_settings()
+
+    def _load_settings(self) -> None:
+        """Load all settings from the registry."""
+        # Ensure all settings modules are imported before loading
+        # This handles cases where config is accessed before settings are registered
+        try:
+            import shelfmark.config.settings  # noqa: F401 - main app settings
+            import shelfmark.release_sources  # noqa: F401 - plugin settings
+            import shelfmark.metadata_providers  # noqa: F401 - plugin settings
+        except ImportError:
+            pass
+
+        registry = _get_registry()
+
+        # On first load, sync ENV values to config files
+        # This ensures ENV values persist even if ENV vars are later removed
+        if not hasattr(self, '_env_synced'):
+            registry.sync_env_to_config()
+            self._env_synced = True
+
+        # Build field map from all registered tabs
+        self._field_map.clear()
+        self._cache.clear()
+
+        for tab in registry.get_all_settings_tabs():
+            for field in tab.fields:
+                # Skip action buttons and headings - they don't have values
+                if isinstance(field, (registry.ActionButton, registry.HeadingField)):
+                    continue
+
+                key = field.key
+                self._field_map[key] = (field, tab.name)
+
+                # Load current value
+                value = registry.get_setting_value(field, tab.name)
+                self._cache[key] = value
+
+        self._loaded = True
+
+    def refresh(self) -> None:
+        """
+        Refresh all cached settings from config files.
+
+        Call this after settings are updated via the UI to ensure
+        the config singleton reflects the new values.
+        """
+        with self._cache_lock:
+            self._loaded = False
+            self._load_settings()
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """
+        Get a setting value by key.
+
+        Args:
+            key: The setting key (e.g., 'MAX_RETRY')
+            default: Default value if setting not found
+
+        Returns:
+            The setting value, or default if not found
+        """
+        self._ensure_loaded()
+        return self._cache.get(key, default)
+
+    def __getattr__(self, name: str) -> Any:
+        """
+        Allow attribute-style access to settings.
+
+        Example: config.MAX_RETRY instead of config.get('MAX_RETRY')
+        """
+        # Avoid recursion for internal attributes
+        if name.startswith('_'):
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
+
+        self._ensure_loaded()
+
+        if name in self._cache:
+            return self._cache[name]
+
+        # Fallback to env module for settings not in registry
+        # This ensures backward compatibility during migration
+        env = _get_env()
+        if hasattr(env, name):
+            return getattr(env, name)
+
+        raise AttributeError(f"Setting '{name}' not found in config or env")
+
+    def is_from_env(self, key: str) -> bool:
+        """
+        Check if a setting's value comes from an environment variable.
+
+        Args:
+            key: The setting key
+
+        Returns:
+            True if the value is set via ENV var, False otherwise
+        """
+        self._ensure_loaded()
+
+        if key not in self._field_map:
+            return False
+
+        field, _ = self._field_map[key]
+        registry = _get_registry()
+        return registry.is_value_from_env(field)
+
+    def get_all(self) -> Dict[str, Any]:
+        """
+        Get all cached settings as a dictionary.
+
+        Returns:
+            Dict of all setting keys to their current values
+        """
+        self._ensure_loaded()
+        return dict(self._cache)
+
+
+# Global singleton instance
+config = Config()

shelfmark/core/image_cache.py

@@ -0,0 +1,569 @@
+"""Disk-based image cache with LRU eviction."""
+
+import json
+import os
+import threading
+import time
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Image type detection via magic bytes
+IMAGE_SIGNATURES = {
+    b'\xff\xd8\xff': ('image/jpeg', 'jpg'),
+    b'\x89PNG\r\n\x1a\n': ('image/png', 'png'),
+    b'GIF87a': ('image/gif', 'gif'),
+    b'GIF89a': ('image/gif', 'gif'),
+    b'RIFF': ('image/webp', 'webp'),  # WebP starts with RIFF
+}
+
+# HTTP headers for image fetching
+FETCH_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36',
+    'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+}
+
+# Maximum image size to fetch (5 MB)
+MAX_IMAGE_SIZE = 5 * 1024 * 1024
+
+# Negative cache TTL (for failed fetches) - 1 hour
+NEGATIVE_CACHE_TTL = 3600
+
+# Transient failure cache TTL (for timeouts/connection errors) - 60 seconds
+# Short enough to retry soon, long enough to prevent spam during one page view
+TRANSIENT_CACHE_TTL = 60
+
+
+def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
+    """Detect image type from magic bytes.
+
+    Args:
+        data: Image data bytes
+
+    Returns:
+        Tuple of (content_type, extension) or None if not recognized
+    """
+    for signature, (content_type, ext) in IMAGE_SIGNATURES.items():
+        if data.startswith(signature):
+            return content_type, ext
+
+    # Special case for WebP - check for WEBP after RIFF
+    if data.startswith(b'RIFF') and len(data) > 12 and data[8:12] == b'WEBP':
+        return 'image/webp', 'webp'
+
+    return None
+
+
+class ImageCacheService:
+    """Persistent image cache with LRU eviction and TTL support."""
+
+    def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0):
+        """Initialize the image cache.
+
+        Args:
+            cache_dir: Directory to store cached images
+            max_size_mb: Maximum cache size in megabytes
+            ttl_seconds: Time-to-live in seconds (0 = forever)
+        """
+        self.cache_dir = cache_dir
+        self.max_size_bytes = max_size_mb * 1024 * 1024
+        self.ttl_seconds = ttl_seconds
+        self.index_path = cache_dir / "cache_index.json"
+        self._lock = threading.RLock()
+        self._index: Dict[str, Dict[str, Any]] = {}
+
+        # Stats tracking
+        self._hits = 0
+        self._misses = 0
+
+        # Ensure cache directory exists
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load existing index and sync with files on disk (once at startup)
+        self._load_index()
+        self._sync_index_with_files()
+
+    def _load_index(self) -> None:
+        """Load cache index from disk."""
+        if not self.index_path.exists():
+            self._index = {}
+            return
+
+        try:
+            with open(self.index_path, 'r') as f:
+                self._index = json.load(f)
+        except (json.JSONDecodeError, IOError):
+            self._index = {}
+
+    def _sync_index_with_files(self) -> None:
+        """Sync cache index with actual files on disk.
+
+        - Adds entries for files that exist but aren't in index
+        - Removes entries for files that no longer exist (non-negative only)
+        - Preserves negative cache entries (they have no files)
+        """
+        image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
+        added_count = 0
+        removed_count = 0
+
+        # Build set of files that exist on disk
+        existing_files: Dict[str, Path] = {}
+        for file_path in self.cache_dir.iterdir():
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() not in image_extensions:
+                continue
+            existing_files[file_path.stem] = file_path
+
+        # Add files that aren't in the index
+        for cache_id, file_path in existing_files.items():
+            if cache_id in self._index:
+                continue
+
+            ext = file_path.suffix.lstrip('.')
+            stat = file_path.stat()
+
+            # Detect content type
+            try:
+                with open(file_path, 'rb') as f:
+                    header = f.read(16)
+                detected = _detect_image_type(header)
+                content_type = detected[0] if detected else f'image/{ext}'
+            except IOError:
+                content_type = f'image/{ext}'
+
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': stat.st_size,
+                'cached_at': stat.st_mtime,
+                'accessed_at': stat.st_mtime,
+            }
+            added_count += 1
+
+        # Remove index entries for missing files (skip negative cache entries)
+        stale_entries = []
+        for cache_id, entry in self._index.items():
+            if entry.get('negative', False):
+                continue  # Negative entries don't have files
+            if cache_id not in existing_files:
+                stale_entries.append(cache_id)
+
+        for cache_id in stale_entries:
+            del self._index[cache_id]
+            removed_count += 1
+
+        if added_count > 0 or removed_count > 0:
+            self._save_index()
+
+    def _save_index(self) -> None:
+        """Save cache index to disk."""
+        try:
+            # Write to temp file first, then rename for atomicity
+            temp_path = self.index_path.with_suffix('.tmp')
+            with open(temp_path, 'w') as f:
+                json.dump(self._index, f)
+            temp_path.rename(self.index_path)
+        except IOError:
+            pass
+
+    def _get_image_path(self, cache_id: str, ext: str) -> Path:
+        """Get the file path for a cached image."""
+        return self.cache_dir / f"{cache_id}.{ext}"
+
+    def _is_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a cache entry is expired."""
+        if self.ttl_seconds == 0:
+            return False
+        return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
+
+    def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a negative cache entry is expired.
+
+        Transient failures (timeouts) expire after TRANSIENT_CACHE_TTL (60s).
+        Permanent failures (404s) expire after NEGATIVE_CACHE_TTL (1 hour).
+        """
+        if not entry.get('negative', False):
+            return False
+
+        cached_at = entry.get('cached_at', 0)
+        ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
+        return (time.time() - cached_at) > ttl
+
+    def _calculate_total_size(self) -> int:
+        """Calculate total size of cached images."""
+        return sum(entry.get('size', 0) for entry in self._index.values())
+
+    def _evict_if_needed(self, required_space: int = 0) -> None:
+        """Evict old entries if cache is over size limit.
+
+        Uses LRU eviction based on accessed_at timestamp.
+        """
+        current_size = self._calculate_total_size()
+        target_size = self.max_size_bytes - required_space
+
+        if current_size <= target_size:
+            return
+
+        # Sort entries by accessed_at (oldest first)
+        sorted_entries = sorted(
+            self._index.items(),
+            key=lambda x: x[1].get('accessed_at', 0)
+        )
+
+        evicted_count = 0
+        for cache_id, entry in sorted_entries:
+            if current_size <= target_size:
+                break
+
+            # Delete the image file
+            ext = entry.get('ext', 'jpg')
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                if image_path.exists():
+                    image_path.unlink()
+            except IOError:
+                pass
+
+            # Update tracking
+            current_size -= entry.get('size', 0)
+            del self._index[cache_id]
+            evicted_count += 1
+
+        if evicted_count > 0:
+            self._save_index()
+
+    def get(self, cache_id: str) -> Optional[Tuple[bytes, str]]:
+        """Get a cached image.
+
+        Args:
+            cache_id: Cache key (book ID or composite key)
+
+        Returns:
+            Tuple of (image_data, content_type) or None if not cached/expired
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+
+            # Try reloading from disk if not found (handles multiprocess case)
+            if not entry:
+                self._load_index()
+                entry = self._index.get(cache_id)
+                if not entry:
+                    self._misses += 1
+                    return None
+
+            # Check for negative cache (failed fetch)
+            if entry.get('negative', False):
+                if self._is_negative_expired(entry):
+                    # Negative cache expired, allow retry
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+                # Still in negative cache, return None (don't retry)
+                return None
+
+            # Check for expired entry
+            if self._is_expired(entry):
+                # Remove expired entry
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+                del self._index[cache_id]
+                self._save_index()
+                self._misses += 1
+                return None
+
+            # Try to read the cached image
+            ext = entry.get('ext', 'jpg')
+            content_type = entry.get('content_type', 'image/jpeg')
+            image_path = self._get_image_path(cache_id, ext)
+
+            try:
+                if not image_path.exists():
+                    # File missing, remove from index
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+
+                with open(image_path, 'rb') as f:
+                    data = f.read()
+
+                # Update accessed time
+                entry['accessed_at'] = time.time()
+                self._save_index()
+
+                self._hits += 1
+                return data, content_type
+
+            except IOError:
+                self._misses += 1
+                return None
+
+    def put(self, cache_id: str, data: bytes, content_type: str) -> bool:
+        """Store an image in the cache.
+
+        Args:
+            cache_id: Cache key
+            data: Image data bytes
+            content_type: MIME type of the image
+
+        Returns:
+            True if stored successfully
+        """
+        with self._lock:
+            # Detect image type for extension
+            detected = _detect_image_type(data)
+            if detected:
+                content_type, ext = detected
+            else:
+                # Fall back to content-type header
+                if 'jpeg' in content_type or 'jpg' in content_type:
+                    ext = 'jpg'
+                elif 'png' in content_type:
+                    ext = 'png'
+                elif 'gif' in content_type:
+                    ext = 'gif'
+                elif 'webp' in content_type:
+                    ext = 'webp'
+                else:
+                    ext = 'jpg'  # Default
+
+            image_size = len(data)
+
+            # Evict if needed to make room
+            self._evict_if_needed(image_size)
+
+            # Write image to disk
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                with open(image_path, 'wb') as f:
+                    f.write(data)
+            except IOError:
+                return False
+
+            # Update index
+            now = time.time()
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': image_size,
+                'cached_at': now,
+                'accessed_at': now,
+                'negative': False,
+            }
+            self._save_index()
+            return True
+
+    def put_negative(self, cache_id: str, transient: bool = False) -> None:
+        """Store a negative cache entry (failed fetch).
+
+        Args:
+            cache_id: Cache key
+            transient: If True, uses shorter TTL (for timeouts/connection errors)
+        """
+        with self._lock:
+            self._index[cache_id] = {
+                'negative': True,
+                'transient': transient,
+                'cached_at': time.time(),
+            }
+            self._save_index()
+
+    def delete(self, cache_id: str) -> bool:
+        """Delete a single cache entry.
+
+        Args:
+            cache_id: Cache key
+
+        Returns:
+            True if entry existed and was deleted
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+            if not entry:
+                return False
+
+            # Delete file if it exists
+            if not entry.get('negative', False):
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+
+            del self._index[cache_id]
+            self._save_index()
+            return True
+
+    def clear(self) -> int:
+        """Clear all cached images.
+
+        Returns:
+            Number of entries cleared
+        """
+        with self._lock:
+            count = len(self._index)
+
+            # Delete all image files
+            for cache_id, entry in self._index.items():
+                if not entry.get('negative', False):
+                    ext = entry.get('ext', 'jpg')
+                    image_path = self._get_image_path(cache_id, ext)
+                    try:
+                        if image_path.exists():
+                            image_path.unlink()
+                    except IOError:
+                        pass
+
+            # Clear index
+            self._index = {}
+            self._save_index()
+
+            # Reset stats
+            self._hits = 0
+            self._misses = 0
+
+            return count
+
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dict with size, count, hit rate, etc.
+        """
+        with self._lock:
+            total_size = self._calculate_total_size()
+            entry_count = len(self._index)
+            negative_count = sum(1 for e in self._index.values() if e.get('negative', False))
+            total_requests = self._hits + self._misses
+            hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0
+
+            return {
+                'entry_count': entry_count,
+                'negative_count': negative_count,
+                'total_size_bytes': total_size,
+                'total_size_mb': round(total_size / (1024 * 1024), 2),
+                'max_size_mb': self.max_size_bytes / (1024 * 1024),
+                'hits': self._hits,
+                'misses': self._misses,
+                'hit_rate': round(hit_rate, 1),
+            }
+
+    def fetch_and_cache(self, cache_id: str, url: str) -> Optional[Tuple[bytes, str]]:
+        """Fetch an image from URL and cache it.
+
+        Args:
+            cache_id: Cache key
+            url: URL to fetch from
+
+        Returns:
+            Tuple of (image_data, content_type) or None on failure
+        """
+        try:
+
+            response = requests.get(
+                url,
+                timeout=(5, 10),
+                headers=FETCH_HEADERS,
+                stream=True,
+            )
+            response.raise_for_status()
+
+            # Validate content type
+            content_type = response.headers.get('content-type', '')
+            if not content_type.startswith('image/'):
+                self.put_negative(cache_id)
+                return None
+
+            # Read with size limit
+            data = BytesIO()
+            for chunk in response.iter_content(chunk_size=8192):
+                data.write(chunk)
+                if data.tell() > MAX_IMAGE_SIZE:
+                    self.put_negative(cache_id)
+                    return None
+
+            image_data = data.getvalue()
+
+            if not image_data:
+                self.put_negative(cache_id)
+                return None
+
+            # Store in cache
+            if self.put(cache_id, image_data, content_type):
+                # Get the actual content type from detection
+                detected = _detect_image_type(image_data)
+                if detected:
+                    content_type = detected[0]
+                return image_data, content_type
+
+            return None
+
+        except requests.exceptions.Timeout:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.ConnectionError:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.HTTPError as e:
+            is_404 = e.response is not None and e.response.status_code == 404
+            self.put_negative(cache_id, transient=not is_404)
+            return None
+        except Exception:
+            return None
+
+
+# Singleton instance (initialized lazily when config is available)
+_instance: Optional[ImageCacheService] = None
+_instance_lock = threading.Lock()
+
+
+def get_image_cache() -> ImageCacheService:
+    """Get the singleton image cache instance.
+
+    Lazily initializes using config values.
+    """
+    global _instance
+
+    if _instance is None:
+        with _instance_lock:
+            if _instance is None:
+                from shelfmark.core.config import config
+                from shelfmark.config.env import CONFIG_DIR
+
+                cache_dir = CONFIG_DIR / "covers"
+                max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
+                ttl_days = config.get("COVERS_CACHE_TTL", 0)
+                ttl_seconds = ttl_days * 86400 if ttl_days > 0 else 0
+
+                _instance = ImageCacheService(
+                    cache_dir=cache_dir,
+                    max_size_mb=max_size_mb,
+                    ttl_seconds=ttl_seconds,
+                )
+                logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
+
+    return _instance
+
+
+def reset_image_cache() -> None:
+    """Reset the singleton instance (for testing or config changes)."""
+    global _instance
+    with _instance_lock:
+        _instance = None

shelfmark/core/logger.py

@@ -1,72 +1,80 @@
-"""Centralized logging configuration for the book downloader application."""
+"""Logging configuration and custom logger with error tracing."""
 
 import logging
 import sys
 from pathlib import Path
 from logging.handlers import RotatingFileHandler
-from env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
 from typing import Any
 
+from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
+
+
 class CustomLogger(logging.Logger):
     """Custom logger class with additional error_trace method."""
-    
+
     def error_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log an error message with full stack trace."""
         self.log_resource_usage()
+        kwargs.pop('exc_info', None)
         self.error(msg, *args, exc_info=True, **kwargs)
 
     def warning_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log a warning message with full stack trace."""
         self.log_resource_usage()
+        kwargs.pop('exc_info', None)
         self.warning(msg, *args, exc_info=True, **kwargs)
-    
+
     def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log an info message with full stack trace."""
-        self.log_resource_usage()
-        self.info(msg, *args, exc_info=True, **kwargs)
-    
+        """Log an info message (stack trace only if exception active)."""
+        kwargs.pop('exc_info', None)
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.info(msg, *args, exc_info=has_exception, **kwargs)
+
     def debug_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log a debug message with full stack trace."""
-        self.log_resource_usage()
-        self.debug(msg, *args, exc_info=True, **kwargs)
-    
+        """Log a debug message (stack trace only if exception active)."""
+        kwargs.pop('exc_info', None)
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.debug(msg, *args, exc_info=has_exception, **kwargs)
+
     def log_resource_usage(self):
         import psutil
+
+        # Sum RSS of all processes for actual app memory
+        app_memory_mb = 0
+        for proc in psutil.process_iter(['memory_info']):
+            try:
+                if proc.info['memory_info']:
+                    app_memory_mb += proc.info['memory_info'].rss / (1024 * 1024)
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+
         memory = psutil.virtual_memory()
+        system_used_mb = memory.used / (1024 * 1024)
         available_mb = memory.available / (1024 * 1024)
-        memory_used_mb = memory.used / (1024 * 1024)
         cpu_percent = psutil.cpu_percent()
-        self.debug(f"Container Memory: Available={available_mb:.2f} MB, Used={memory_used_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
+        self.debug(f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
 
 
 def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     """Set up and configure a logger instance.
-    
+
     Args:
         name: The name of the logger instance
         log_file: Optional path to log file. If None, logs only to stdout/stderr
-        
+
     Returns:
         CustomLogger: Configured logger instance with error_trace method
     """
     # Register our custom logger class
     logging.setLoggerClass(CustomLogger)
-    
+
     # Create logger as CustomLogger instance
     logger = CustomLogger(name)
-    log_level = logging.INFO
-    if LOG_LEVEL == "DEBUG":
-        log_level = logging.DEBUG
-    elif LOG_LEVEL == "INFO":
-        log_level = logging.INFO
-    elif LOG_LEVEL == "WARNING":
-        log_level = logging.WARNING
-    elif LOG_LEVEL == "ERROR":
-        log_level = logging.ERROR
-    elif LOG_LEVEL == "CRITICAL":
-        log_level = logging.CRITICAL
+    log_level = getattr(logging, LOG_LEVEL, logging.INFO)
     logger.setLevel(log_level)
-    
+
     formatter = logging.Formatter(
         '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
     )
@@ -77,13 +85,13 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     console_handler.setLevel(log_level)
     console_handler.addFilter(lambda record: record.levelno < logging.ERROR)  # Only allow logs below ERROR to stdout
     logger.addHandler(console_handler)
-    
+
     # Error handler for stderr
     error_handler = logging.StreamHandler(sys.stderr)
     error_handler.setLevel(logging.ERROR) # Error and above go to stderr
     error_handler.setFormatter(formatter)
     logger.addHandler(error_handler)
-    
+
     # File handler if log file is specified
     try:
         if ENABLE_LOGGING:
@@ -101,4 +109,3 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
         logger.error_trace(f"Failed to create log file: {e}", exc_info=True)
 
     return logger
-

shelfmark/core/mirrors.py

@@ -0,0 +1,231 @@
+"""Centralized mirror configuration for all download sources."""
+
+from typing import List
+
+from shelfmark.core.utils import normalize_http_url
+
+# Lazy import to avoid circular imports
+_config_module = None
+
+
+def _get_config():
+    """Lazy import of config module to avoid circular imports."""
+    global _config_module
+    if _config_module is None:
+        from shelfmark.core.config import config
+        _config_module = config
+    return _config_module
+
+
+# Default mirror lists (hardcoded fallbacks)
+DEFAULT_AA_MIRRORS = [
+    "https://annas-archive.se",
+    "https://annas-archive.li",
+    "https://annas-archive.pm",
+    "https://annas-archive.in",
+]
+
+DEFAULT_LIBGEN_MIRRORS = [
+    "https://libgen.gl",
+    "https://libgen.li",
+    "https://libgen.bz",
+    "https://libgen.la",
+    "https://libgen.vg",
+]
+
+DEFAULT_ZLIB_MIRRORS = [
+    "https://z-lib.fm",
+    "https://z-lib.gs",
+    "https://z-lib.id",
+    "https://z-library.sk",
+    "https://zlibrary-global.se",
+]
+
+DEFAULT_WELIB_MIRRORS = [
+    "https://welib.org",
+]
+
+
+def _normalize_mirror_url(url: str) -> str:
+    return normalize_http_url(url, default_scheme="https")
+
+
+def get_aa_mirrors() -> List[str]:
+    """
+    Get Anna's Archive mirrors from config + defaults.
+
+    Returns:
+        List of AA mirror URLs, starting with defaults then custom additions.
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("AA_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_libgen_mirrors() -> List[str]:
+    """
+    Get LibGen mirrors: defaults + any additional from config.
+
+    Returns:
+        List of LibGen mirror URLs (defaults first, then custom additions).
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_mirrors() -> List[str]:
+    """
+    Get Z-Library mirrors, with primary first.
+
+    Returns:
+        List of Z-Library mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_primary_url() -> str:
+    """
+    Get the primary Z-Library mirror URL.
+
+    Returns:
+        Primary Z-Library mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+
+
+def get_zlib_url_template() -> str:
+    """
+    Get Z-Library URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_zlib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_welib_mirrors() -> List[str]:
+    """
+    Get Welib mirrors, with primary first.
+
+    Returns:
+        List of Welib mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_WELIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("WELIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_welib_primary_url() -> str:
+    """
+    Get the primary Welib mirror URL.
+
+    Returns:
+        Primary Welib mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+
+
+def get_welib_url_template() -> str:
+    """
+    Get Welib URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_welib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_zlib_cookie_domains() -> set:
+    """
+    Get set of Z-Library domains that need full cookie handling.
+
+    Used by internal_bypasser for CF bypass cookie management.
+
+    Returns:
+        Set of domain strings (without protocol).
+    """
+    domains = set()
+
+    # Add all default domains
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized:
+            domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+            domains.add(domain)
+
+    # Add custom domains
+    config = _get_config()
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized:
+                domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+                domains.add(domain)
+
+    return domains

shelfmark/core/models.py

@@ -0,0 +1,170 @@
+"""Data structures and models used across the application."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional
+from enum import Enum
+import re
+import time
+
+
+def build_filename(
+    title: str,
+    author: Optional[str] = None,
+    year: Optional[str] = None,
+    fmt: Optional[str] = None,
+) -> str:
+    parts = []
+    if author:
+        parts.append(author)
+        parts.append(" - ")
+    parts.append(title)
+    if year:
+        parts.append(f" ({year})")
+
+    filename = "".join(parts)
+    filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
+
+    if fmt:
+        filename = f"{filename}.{fmt}"
+
+    return filename
+
+
+class QueueStatus(str, Enum):
+    """Enum for possible book queue statuses."""
+    QUEUED = "queued"
+    RESOLVING = "resolving"
+    DOWNLOADING = "downloading"
+    COMPLETE = "complete"
+    AVAILABLE = "available"
+    ERROR = "error"
+    DONE = "done"
+    CANCELLED = "cancelled"
+
+
+class SearchMode(str, Enum):
+    DIRECT = "direct"
+    UNIVERSAL = "universal"
+
+
+@dataclass
+class QueueItem:
+    """Queue item with priority and metadata."""
+    book_id: str
+    priority: int
+    added_time: float
+
+    def __lt__(self, other):
+        """Compare items for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+
+@dataclass
+class DownloadTask:
+    task_id: str                                # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
+    source: str                                 # Handler name ("direct_download", "prowlarr")
+    title: str                                  # Display title for queue sidebar
+
+    # Display info for queue sidebar
+    author: Optional[str] = None
+    year: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    preview: Optional[str] = None
+    content_type: Optional[str] = None  # "book (fiction)", "audiobook", "magazine", etc.
+
+    # Series info (for library naming templates)
+    series_name: Optional[str] = None
+    series_position: Optional[float] = None  # Float for novellas (e.g., 1.5)
+    subtitle: Optional[str] = None  # Book subtitle for naming templates
+
+    # Hardlinking support
+    original_download_path: Optional[str] = None  # Path in download client (for hardlinking)
+
+    # Search mode - determines post-download processing behavior
+    # See SearchMode enum for behavioral differences
+    search_mode: Optional[SearchMode] = None
+
+    # Runtime state
+    priority: int = 0
+    added_time: float = field(default_factory=time.time)
+    progress: float = 0.0
+    status: QueueStatus = QueueStatus.QUEUED
+    status_message: Optional[str] = None
+    download_path: Optional[str] = None
+
+    def __lt__(self, other):
+        """Compare tasks for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+    def get_filename(self) -> str:
+        """Build sanitized filename from task metadata."""
+        if self.download_path:
+            return Path(self.download_path).name
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class BookInfo:
+    """Data class representing book information."""
+    id: str
+    title: str
+    preview: Optional[str] = None
+    author: Optional[str] = None
+    publisher: Optional[str] = None
+    year: Optional[str] = None
+    language: Optional[str] = None
+    content: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    info: Optional[Dict[str, List[str]]] = None
+    description: Optional[str] = None
+    download_urls: List[str] = field(default_factory=list)
+    download_path: Optional[str] = None
+    priority: int = 0
+    progress: Optional[float] = None
+    status_message: Optional[str] = None  # Detailed status message for UI display
+    added_time: Optional[float] = None  # Timestamp when added to queue
+    source: str = "direct_download"  # Release source handler to use for downloads
+    source_url: Optional[str] = None  # Link to source page (e.g., Anna's Archive)
+
+    def get_filename(self, fallback_url: Optional[str] = None) -> str:
+        """Build sanitized filename: 'Author - Title (Year).format'
+
+        Resolves format from self.format, download_urls, or fallback_url.
+
+        Args:
+            fallback_url: URL to extract format from if not already known
+
+        Returns:
+            Sanitized filename safe for filesystem use
+        """
+        # Resolve format if needed
+        if not self.format:
+            urls = [self.download_urls[0]] if self.download_urls else []
+            if fallback_url:
+                urls.append(fallback_url)
+            for url in urls:
+                ext = url.split(".")[-1].lower()
+                if ext and len(ext) <= 5 and ext.isalnum():
+                    self.format = ext
+                    break
+
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class SearchFilters:
+    """Filters for book search queries."""
+    isbn: Optional[List[str]] = None
+    author: Optional[List[str]] = None
+    title: Optional[List[str]] = None
+    lang: Optional[List[str]] = None
+    sort: Optional[str] = None
+    content: Optional[List[str]] = None
+    format: Optional[List[str]] = None

shelfmark/core/naming.py

@@ -0,0 +1,201 @@
+"""Template-based naming for library organization."""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional, Union, Mapping
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+TOKEN_PATTERN = re.compile(
+    r'\{([- ._/\[(]*)'   # prefix: space, dash, dot, underscore, slash, brackets
+    r'([A-Za-z]+)'        # token name
+    r'([- ._/\])]*)\}'    # suffix: space, dash, dot, underscore, slash, brackets
+)
+
+# Characters that are invalid in filenames on various filesystems
+INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def _sanitize(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for filesystem use."""
+    if not name:
+        return ""
+
+    sanitized = INVALID_CHARS.sub('_', name)
+    sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized)  # Strip whitespace and dots
+    sanitized = re.sub(r'_+', '_', sanitized)  # Collapse underscores
+    return sanitized[:max_length]
+
+
+def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for use as a filename or path component."""
+    return _sanitize(name, max_length)
+
+
+# Alias for backwards compatibility
+sanitize_path_component = sanitize_filename
+
+
+def format_series_position(position: Optional[Union[str, int, float]]) -> str:
+    if position is None:
+        return ""
+
+    # Display as integer if whole number
+    if isinstance(position, float) and position.is_integer():
+        return str(int(position))
+
+    return str(position)
+
+
+# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
+PAD_NUMBERS_PATTERN = re.compile(r'\d+')
+
+
+def natural_sort_key(path: Union[str, Path]) -> str:
+    """Generate a sort key with padded numbers for natural sorting."""
+    filename = Path(path).name.lower()
+    return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
+
+
+def assign_part_numbers(
+    files: list[Path],
+    zero_pad_width: int = 2,
+) -> list[tuple[Path, str]]:
+    """Sort files naturally and assign sequential part numbers (1, 2, 3...)."""
+    if not files:
+        return []
+
+    sorted_files = sorted(files, key=natural_sort_key)
+    return [
+        (file_path, str(part_num).zfill(zero_pad_width))
+        for part_num, file_path in enumerate(sorted_files, start=1)
+    ]
+
+
+def parse_naming_template(
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    *,
+    allow_path_separators: bool = True,
+) -> str:
+    if not template:
+        return ""
+
+    # Normalize metadata keys to lowercase for case-insensitive matching
+    normalized = {k.lower(): v for k, v in metadata.items()}
+
+    def replace_token(match: re.Match) -> str:
+        prefix = match.group(1)
+        token_name = match.group(2).lower()
+        suffix = match.group(3)
+
+        # Get the value for this token
+        value = normalized.get(token_name)
+
+        # Special handling for series position
+        if token_name == 'seriesposition':
+            value = format_series_position(value)
+
+        # Convert to string
+        if value is None:
+            value = ""
+        else:
+            value = str(value).strip()
+
+        # If value is empty, return empty string (no prefix/suffix)
+        if not value:
+            return ""
+
+        if not allow_path_separators:
+            value = value.replace("/", "_")
+        # Sanitize the value
+        value = sanitize_filename(value)
+
+        return f"{prefix}{value}{suffix}"
+
+    # Replace all tokens
+    result = TOKEN_PATTERN.sub(replace_token, template)
+
+    # Clean up any double slashes that might result from empty tokens
+    result = re.sub(r'/+', '/', result)
+
+    # Remove leading/trailing slashes
+    result = result.strip('/')
+
+    # Clean up any orphaned separators (e.g., " - " at start/end, or " -  - ")
+    result = re.sub(r'^[\s\-_.]+', '', result)
+    result = re.sub(r'[\s\-_.]+$', '', result)
+    result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
+
+    # Clean up empty parentheses/brackets
+    result = re.sub(r'\(\s*\)', '', result)
+    result = re.sub(r'\[\s*\]', '', result)
+
+    # Final trim of any trailing separators left after cleanup
+    result = re.sub(r'[\s\-_.]+$', '', result)
+
+    return result
+
+
+def build_library_path(
+    base_path: str,
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    extension: Optional[str] = None,
+) -> Path:
+    relative = parse_naming_template(template, metadata, allow_path_separators=True)
+
+    if not relative:
+        # Fallback to title if template produces empty result
+        title = metadata.get('Title') or metadata.get('title') or 'Unknown'
+        relative = sanitize_filename(str(title))
+
+    # Remove any path traversal attempts
+    relative = relative.replace('..', '')
+
+    base = Path(base_path).resolve()
+    full_path = (base / relative).resolve()
+
+    # Verify the path is within the base directory
+    try:
+        full_path.relative_to(base)
+    except ValueError:
+        raise ValueError(f"Path traversal detected: template would escape library directory")
+
+    if extension:
+        ext = extension.lstrip('.')
+        # Don't use with_suffix() - it replaces everything after the first dot
+        # e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
+        full_path = Path(f"{full_path}.{ext}")
+
+    return full_path
+
+
+def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
+    """Check if two paths are on the same filesystem."""
+    path1 = Path(path1)
+    path2 = Path(path2)
+
+    def get_device(p: Path) -> Optional[int]:
+        try:
+            while not p.exists():
+                p = p.parent
+                if p == p.parent:
+                    break
+            return os.stat(p).st_dev
+        except (OSError, PermissionError) as e:
+            logger.debug(f"Cannot stat {p}: {e}")
+            return None
+
+    dev1 = get_device(path1)
+    dev2 = get_device(path2)
+
+    if dev1 is None or dev2 is None:
+        logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
+        return False
+
+    return dev1 == dev2

shelfmark/core/onboarding.py

@@ -0,0 +1,429 @@
+"""
+Onboarding wizard configuration.
+
+Defines the steps and fields for the first-run onboarding experience.
+Reuses field definitions from the settings registry where possible.
+"""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    HeadingField,
+    SettingsField,
+    get_settings_tab,
+    serialize_field,
+    save_config_file,
+    get_setting_value,
+)
+
+logger = setup_logger(__name__)
+
+
+ONBOARDING_STORAGE_KEY = "onboarding_complete"
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def is_onboarding_complete() -> bool:
+    """Check if onboarding has been completed."""
+    config_file = _get_config_dir() / "settings.json"
+    if not config_file.exists():
+        return False
+
+    try:
+        with open(config_file, 'r') as f:
+            config = json.load(f)
+            return config.get(ONBOARDING_STORAGE_KEY, False)
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning(f"Could not read onboarding status from settings.json: {e}")
+        return False
+
+
+def mark_onboarding_complete() -> bool:
+    """Mark onboarding as complete."""
+    try:
+        return save_config_file("general", {ONBOARDING_STORAGE_KEY: True})
+    except Exception as e:
+        logger.error(f"Failed to mark onboarding complete: {e}")
+        return False
+
+
+def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField]:
+    """
+    Extract a specific field from a registered settings tab.
+
+    Args:
+        tab_name: Name of the settings tab (e.g., 'search_mode', 'hardcover')
+        field_key: Key of the field to extract (e.g., 'SEARCH_MODE', 'HARDCOVER_API_KEY')
+
+    Returns:
+        The field if found, None otherwise
+    """
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        logger.warning(f"Settings tab not found: {tab_name}")
+        return None
+
+    for field in tab.fields:
+        if hasattr(field, 'key') and field.key == field_key:
+            return field
+
+    logger.warning(f"Field {field_key} not found in tab {tab_name}")
+    return None
+
+
+def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
+    """
+    Clone a field with optional attribute overrides.
+
+    Useful for customizing labels, descriptions, or defaults for onboarding context.
+    """
+    return replace(field, **overrides)
+
+
+# =============================================================================
+# Step Definitions
+# =============================================================================
+
+
+def get_search_mode_fields() -> List[SettingsField]:
+    """Step 1: Choose search mode - uses actual SEARCH_MODE field from settings."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="welcome_heading",
+            title="Welcome to Shelfmark",
+            description="Let's configure how you want to search for and download books.",
+        ),
+    ]
+
+    # Get the actual SEARCH_MODE field from settings
+    search_mode_field = _get_field_from_tab("search_mode", "SEARCH_MODE")
+    if search_mode_field:
+        # Clone with onboarding-specific description
+        fields.append(_clone_field_with_overrides(
+            search_mode_field,
+            description="Choose how you want to find books.",
+        ))
+
+    return fields
+
+
+def get_metadata_provider_fields() -> List[SettingsField]:
+    """Step 2: Choose metadata provider - uses actual METADATA_PROVIDER field."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="metadata_heading",
+            title="Metadata Provider",
+            description="Choose where to search for book information. You can enable more providers in Settings later.",
+        ),
+    ]
+
+    # Get the actual METADATA_PROVIDER field from settings
+    provider_field = _get_field_from_tab("search_mode", "METADATA_PROVIDER")
+    if provider_field:
+        # Custom options with Hardcover marked as recommended
+        onboarding_options = [
+            {
+                "value": "hardcover",
+                "label": "Hardcover (Recommended)",
+                "description": "Modern book tracking platform with excellent metadata, ratings, and series information. Requires free API key.",
+            },
+            {
+                "value": "openlibrary",
+                "label": "Open Library",
+                "description": "Free, open-source library catalog from the Internet Archive. No API key required.",
+            },
+            {
+                "value": "googlebooks",
+                "label": "Google Books",
+                "description": "Google's book database with good coverage. Requires free API key.",
+            },
+        ]
+
+        # Clone with onboarding-specific options and default
+        fields.append(_clone_field_with_overrides(
+            provider_field,
+            default="hardcover",
+            options=onboarding_options,
+        ))
+
+    return fields
+
+
+def get_hardcover_setup_fields() -> List[SettingsField]:
+    """Step 3a: Configure Hardcover - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="hardcover_setup_heading",
+            title="Hardcover Setup",
+            description="Get your free API key from hardcover.app/account/api",
+            link_url="https://hardcover.app/account/api",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual HARDCOVER_API_KEY field
+    api_key_field = _get_field_from_tab("hardcover", "HARDCOVER_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("hardcover", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_googlebooks_setup_fields() -> List[SettingsField]:
+    """Step 3b: Configure Google Books - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="googlebooks_setup_heading",
+            title="Google Books Setup",
+            description="Get your free API key from Google Cloud Console (APIs & Services > Credentials).",
+            link_url="https://console.cloud.google.com/apis/library/books.googleapis.com",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual GOOGLEBOOKS_API_KEY field
+    api_key_field = _get_field_from_tab("googlebooks", "GOOGLEBOOKS_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("googlebooks", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_prowlarr_fields() -> List[SettingsField]:
+    """Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_heading",
+            title="Prowlarr Integration (Optional)",
+            description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.",
+        ),
+    ]
+
+    # Get actual Prowlarr connection fields
+    prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"]
+    for field_key in prowlarr_fields:
+        field = _get_field_from_tab("prowlarr_config", field_key)
+        if field:
+            fields.append(field)
+
+    return fields
+
+
+def get_prowlarr_indexers_fields() -> List[SettingsField]:
+    """Step 5: Select Prowlarr indexers to search."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_indexers_heading",
+            title="Select Indexers",
+            description="Choose which indexers to search for books. Leave empty to search all available indexers.",
+        ),
+    ]
+
+    # Get the indexers multi-select field
+    indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS")
+    if indexers_field:
+        fields.append(indexers_field)
+
+    return fields
+
+
+# =============================================================================
+# Step Configuration
+# =============================================================================
+
+
+ONBOARDING_STEPS = [
+    {
+        "id": "search_mode",
+        "title": "Search Mode",
+        "tab": "search_mode",
+        "get_fields": get_search_mode_fields,
+    },
+    {
+        "id": "metadata_provider",
+        "title": "Metadata Provider",
+        "tab": "search_mode",
+        "get_fields": get_metadata_provider_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+    },
+    {
+        "id": "hardcover_setup",
+        "title": "Hardcover Setup",
+        "tab": "hardcover",
+        "get_fields": get_hardcover_setup_fields,
+        # Must be universal mode AND hardcover selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "hardcover"},
+        ],
+    },
+    {
+        "id": "googlebooks_setup",
+        "title": "Google Books Setup",
+        "tab": "googlebooks",
+        "get_fields": get_googlebooks_setup_fields,
+        # Must be universal mode AND googlebooks selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "googlebooks"},
+        ],
+    },
+    {
+        "id": "prowlarr",
+        "title": "Prowlarr",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+        "optional": True,
+    },
+    {
+        "id": "prowlarr_indexers",
+        "title": "Indexers",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_indexers_fields,
+        # Only show when Prowlarr is enabled
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "PROWLARR_ENABLED", "value": True},
+        ],
+        "optional": True,
+    },
+]
+
+
+def get_onboarding_config() -> Dict[str, Any]:
+    """
+    Get the full onboarding configuration including steps and current values.
+    """
+    steps = []
+    all_values = {}
+
+    for step_config in ONBOARDING_STEPS:
+        fields = step_config["get_fields"]()
+        tab_name = step_config["tab"]
+
+        # Serialize fields with current values
+        serialized_fields = []
+        for field in fields:
+            serialized = serialize_field(field, tab_name, include_value=True)
+            serialized_fields.append(serialized)
+
+            # Collect values (skip HeadingFields)
+            if hasattr(field, 'key') and field.key and not isinstance(field, HeadingField):
+                value = get_setting_value(field, tab_name)
+                all_values[field.key] = value if value is not None else getattr(field, 'default', '')
+
+        step = {
+            "id": step_config["id"],
+            "title": step_config["title"],
+            "tab": tab_name,
+            "fields": serialized_fields,
+        }
+
+        if "show_when" in step_config:
+            step["showWhen"] = step_config["show_when"]
+        if step_config.get("optional"):
+            step["optional"] = True
+
+        steps.append(step)
+
+    return {
+        "steps": steps,
+        "values": all_values,
+        "complete": is_onboarding_complete(),
+    }
+
+
+def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Save onboarding settings and mark as complete.
+
+    Args:
+        values: Dict of field key -> value
+
+    Returns:
+        Dict with success status and message
+    """
+    try:
+        # Group values by their target tab
+        tab_values: Dict[str, Dict[str, Any]] = {}
+
+        for step_config in ONBOARDING_STEPS:
+            tab_name = step_config["tab"]
+            fields = step_config["get_fields"]()
+
+            for field in fields:
+                if isinstance(field, HeadingField):
+                    continue
+
+                key = field.key
+                if key in values:
+                    if tab_name not in tab_values:
+                        tab_values[tab_name] = {}
+                    tab_values[tab_name][key] = values[key]
+
+        # Save each tab's values
+        for tab_name, tab_data in tab_values.items():
+            if tab_data:
+                save_config_file(tab_name, tab_data)
+                logger.info(f"Saved onboarding settings to {tab_name}: {list(tab_data.keys())}")
+
+        # Enable the selected metadata provider
+        search_mode = values.get("SEARCH_MODE", "direct")
+        if search_mode == "universal":
+            provider = values.get("METADATA_PROVIDER", "hardcover")
+            if provider:
+                # Map provider name to its enabled key
+                enabled_key_map = {
+                    "hardcover": "HARDCOVER_ENABLED",
+                    "openlibrary": "OPENLIBRARY_ENABLED",
+                    "googlebooks": "GOOGLEBOOKS_ENABLED",
+                }
+                enabled_key = enabled_key_map.get(provider, f"{provider.upper()}_ENABLED")
+
+                # Get existing provider config and add enabled flag
+                provider_config = {enabled_key: True}
+
+                # Include API key if provided for that provider
+                if provider == "hardcover" and values.get("HARDCOVER_API_KEY"):
+                    provider_config["HARDCOVER_API_KEY"] = values["HARDCOVER_API_KEY"]
+                elif provider == "googlebooks" and values.get("GOOGLEBOOKS_API_KEY"):
+                    provider_config["GOOGLEBOOKS_API_KEY"] = values["GOOGLEBOOKS_API_KEY"]
+
+                save_config_file(provider, provider_config)
+                logger.info(f"Enabled metadata provider: {provider} with keys: {list(provider_config.keys())}")
+
+        # Mark onboarding as complete
+        mark_onboarding_complete()
+
+        # Refresh config
+        try:
+            from shelfmark.core.config import config
+            config.refresh()
+        except ImportError as e:
+            logger.debug(f"Could not refresh config after onboarding: {e}")
+
+        return {"success": True, "message": "Onboarding complete!"}
+
+    except Exception as e:
+        logger.error(f"Failed to save onboarding settings: {e}")
+        return {"success": False, "message": str(e)}

shelfmark/core/path_mappings.py

@@ -0,0 +1,136 @@
+"""Remote path mapping utilities.
+
+Used when an external download client reports a completed download path that does
+not exist inside the Shelfmark runtime environment (commonly different Docker
+volume mounts).
+
+A mapping rewrites a remote path prefix into a local path prefix.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+
+@dataclass(frozen=True)
+class RemotePathMapping:
+    host: str
+    remote_path: str
+    local_path: str
+
+
+def _normalize_prefix(path: str) -> str:
+    normalized = str(path or "").strip()
+    if not normalized:
+        return ""
+
+    normalized = normalized.replace("\\", "/")
+
+    if normalized != "/":
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def _is_windows_path(path: str) -> bool:
+    """Check if a path looks like a Windows path (has a drive letter like C:/)."""
+    return len(path) >= 2 and path[1] == ":" and path[0].isalpha()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host or "").strip().lower()
+
+
+def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
+    if not value or not isinstance(value, list):
+        return []
+
+    mappings: list[RemotePathMapping] = []
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        host = _normalize_host(row.get("host", ""))
+        remote_path = _normalize_prefix(row.get("remotePath", ""))
+        local_path = _normalize_prefix(row.get("localPath", ""))
+
+        if not host or not remote_path or not local_path:
+            continue
+
+        mappings.append(RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path))
+
+    mappings.sort(key=lambda m: len(m.remote_path), reverse=True)
+    return mappings
+
+
+def remap_remote_to_local_with_match(
+    *,
+    mappings: Iterable[RemotePathMapping],
+    host: str,
+    remote_path: str | Path,
+) -> tuple[Path, bool]:
+    host_normalized = _normalize_host(host)
+    remote_normalized = _normalize_prefix(str(remote_path))
+
+    if not remote_normalized:
+        return Path(str(remote_path)), False
+
+    # Windows paths are case-insensitive, so we need case-insensitive matching
+    # for paths that look like Windows paths (e.g., D:/Torrents)
+    is_windows = _is_windows_path(remote_normalized)
+
+    for mapping in mappings:
+        if _normalize_host(mapping.host) != host_normalized:
+            continue
+
+        remote_prefix = _normalize_prefix(mapping.remote_path)
+        if not remote_prefix:
+            continue
+
+        # For Windows paths, do case-insensitive prefix matching
+        if is_windows:
+            remote_lower = remote_normalized.lower()
+            prefix_lower = remote_prefix.lower()
+            matches = remote_lower == prefix_lower or remote_lower.startswith(prefix_lower + "/")
+        else:
+            matches = remote_normalized == remote_prefix or remote_normalized.startswith(remote_prefix + "/")
+
+        if matches:
+            # Use the length of the original prefix to extract remainder
+            # This preserves the original case in folder names
+            remainder = remote_normalized[len(remote_prefix):]
+            local_prefix = _normalize_prefix(mapping.local_path)
+
+            if remainder.startswith("/"):
+                remainder = remainder[1:]
+
+            remapped = Path(local_prefix) / remainder if remainder else Path(local_prefix)
+            return remapped, True
+
+    return Path(remote_normalized), False
+
+
+def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path) -> Path:
+    remapped, _ = remap_remote_to_local_with_match(
+        mappings=mappings,
+        host=host,
+        remote_path=remote_path,
+    )
+    return remapped
+
+
+def get_client_host_identifier(client: Any) -> Optional[str]:
+    """Return a stable identifier used by the mapping UI.
+
+    Sonarr uses the download client's configured host. Shelfmark currently uses
+    the download client 'name' (e.g. qbittorrent, sabnzbd).
+    """
+
+    name = getattr(client, "name", None)
+    if isinstance(name, str) and name.strip():
+        return name.strip().lower()
+
+    return None

shelfmark/core/prefix_middleware.py

@@ -0,0 +1,31 @@
+"""WSGI middleware for hosting Shelfmark under a URL prefix."""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+
+class PrefixMiddleware:
+    """Strip a configured URL prefix from PATH_INFO before routing."""
+
+    def __init__(self, app, prefix: str, bypass_paths: Optional[Iterable[str]] = None) -> None:
+        self.app = app
+        self.prefix = prefix.rstrip("/")
+        self.bypass_paths = set(bypass_paths or [])
+
+    def __call__(self, environ, start_response):
+        path = environ.get("PATH_INFO", "") or ""
+
+        if path in self.bypass_paths:
+            return self.app(environ, start_response)
+
+        if not self.prefix:
+            return self.app(environ, start_response)
+
+        if path == self.prefix or path.startswith(self.prefix + "/"):
+            environ["SCRIPT_NAME"] = self.prefix
+            environ["PATH_INFO"] = path[len(self.prefix):] or "/"
+            return self.app(environ, start_response)
+
+        start_response("404 Not Found", [("Content-Type", "text/plain")])
+        return [b"Not Found"]

shelfmark/core/queue.py

@@ -0,0 +1,296 @@
+"""Thread-safe download queue manager with priority support and cancellation."""
+
+import queue
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from threading import Lock, Event
+from typing import Dict, List, Optional, Tuple, Any
+
+from shelfmark.core.config import config as app_config
+from shelfmark.core.models import QueueStatus, QueueItem, DownloadTask
+
+
+class BookQueue:
+    """Thread-safe download queue manager with priority support and cancellation."""
+
+    def __init__(self) -> None:
+        self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
+        self._lock = Lock()
+        self._status: dict[str, QueueStatus] = {}
+        self._task_data: dict[str, DownloadTask] = {}
+        self._status_timestamps: dict[str, datetime] = {}  # Track when each status was last updated
+        self._cancel_flags: dict[str, Event] = {}  # Cancellation flags for active downloads
+        self._active_downloads: dict[str, bool] = {}  # Track currently downloading tasks
+
+    @property
+    def _status_timeout(self) -> timedelta:
+        """Get status timeout from config (allows live updates)."""
+        return timedelta(seconds=app_config.get("STATUS_TIMEOUT", 3600))
+
+    def add(self, task: DownloadTask) -> bool:
+        """Add a download task to the queue. Returns False if already exists."""
+        with self._lock:
+            task_id = task.task_id
+
+            # Don't add if already exists and not in error/done state
+            if task_id in self._status and self._status[task_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
+                return False
+
+            # Ensure added_time is set
+            if task.added_time == 0:
+                task.added_time = time.time()
+
+            queue_item = QueueItem(task_id, task.priority, task.added_time)
+            self._queue.put(queue_item)
+            self._task_data[task_id] = task
+            self._update_status(task_id, QueueStatus.QUEUED)
+            return True
+
+    def get_next(self) -> Optional[Tuple[str, Event]]:
+        """Get next task ID from queue with cancellation flag."""
+        # Use iterative approach to avoid stack overflow if many items are cancelled
+        while True:
+            try:
+                queue_item = self._queue.get_nowait()
+                task_id = queue_item.book_id  # QueueItem uses book_id as the ID field
+
+                with self._lock:
+                    # Check if task was cancelled while in queue
+                    if task_id in self._status and self._status[task_id] == QueueStatus.CANCELLED:
+                        continue  # Skip cancelled items, try next
+
+                    # Create cancellation flag for this download
+                    cancel_flag = Event()
+                    self._cancel_flags[task_id] = cancel_flag
+                    self._active_downloads[task_id] = True
+
+                return task_id, cancel_flag
+            except queue.Empty:
+                return None
+
+    def get_task(self, task_id: str) -> Optional[DownloadTask]:
+        """Get a task by its ID."""
+        with self._lock:
+            return self._task_data.get(task_id)
+
+    def _update_status(self, book_id: str, status: QueueStatus) -> None:
+        """Internal method to update status and timestamp."""
+        self._status[book_id] = status
+        self._status_timestamps[book_id] = datetime.now()
+
+    def update_status(self, book_id: str, status: QueueStatus) -> None:
+        """Update status of a book in the queue."""
+        with self._lock:
+            self._update_status(book_id, status)
+
+            # Clean up active download tracking when finished
+            if status in [QueueStatus.COMPLETE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
+                self._active_downloads.pop(book_id, None)
+                self._cancel_flags.pop(book_id, None)
+
+    def update_download_path(self, task_id: str, download_path: str) -> None:
+        """Update the download path of a task in the queue."""
+        with self._lock:
+            if task_id in self._task_data:
+                self._task_data[task_id].download_path = download_path
+
+    def update_progress(self, task_id: str, progress: float) -> None:
+        """Update download progress for a task."""
+        with self._lock:
+            if task_id in self._task_data:
+                self._task_data[task_id].progress = progress
+
+    def update_status_message(self, task_id: str, message: str) -> None:
+        """Update detailed status message for a task."""
+        with self._lock:
+            if task_id in self._task_data:
+                self._task_data[task_id].status_message = message
+
+    def get_status(self) -> Dict[QueueStatus, Dict[str, DownloadTask]]:
+        """Get current queue status grouped by status."""
+        self.refresh()
+        with self._lock:
+            result: Dict[QueueStatus, Dict[str, DownloadTask]] = {status: {} for status in QueueStatus}
+            for task_id, status in self._status.items():
+                if task_id in self._task_data:
+                    result[status][task_id] = self._task_data[task_id]
+            return result
+
+    def get_queue_order(self) -> List[Dict[str, Any]]:
+        """Get current queue order for display."""
+        with self._lock:
+            queue_items = []
+
+            # Get items from priority queue without removing them
+            temp_items = []
+            while not self._queue.empty():
+                try:
+                    item = self._queue.get_nowait()
+                    temp_items.append(item)
+                    task_id = item.book_id  # QueueItem uses book_id as the ID field
+                    if task_id in self._task_data:
+                        task = self._task_data[task_id]
+                        queue_items.append({
+                            'id': task_id,
+                            'title': task.title,
+                            'author': task.author,
+                            'priority': item.priority,
+                            'added_time': item.added_time,
+                            'status': self._status.get(task_id, QueueStatus.QUEUED)
+                        })
+                except queue.Empty:
+                    break
+
+            # Put items back in queue
+            for item in temp_items:
+                self._queue.put(item)
+
+            return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
+
+    def cancel_download(self, task_id: str) -> bool:
+        """Cancel a download or clear a completed/errored item."""
+        with self._lock:
+            current_status = self._status.get(task_id)
+
+            # Allow cancellation during any active state
+            if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]:
+                # Signal active download to stop
+                if task_id in self._cancel_flags:
+                    self._cancel_flags[task_id].set()
+                self._update_status(task_id, QueueStatus.CANCELLED)
+                return True
+            elif current_status == QueueStatus.QUEUED:
+                # Remove from queue and mark as cancelled
+                self._update_status(task_id, QueueStatus.CANCELLED)
+                return True
+            elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
+                # Clear completed/errored/cancelled items from tracking
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
+                self._cancel_flags.pop(task_id, None)
+                self._active_downloads.pop(task_id, None)
+                return True
+
+            return False
+
+    def set_priority(self, task_id: str, new_priority: int) -> bool:
+        """Change the priority of a queued task (lower = higher priority)."""
+        with self._lock:
+            if task_id not in self._status or self._status[task_id] != QueueStatus.QUEUED:
+                return False
+
+            # Remove task from queue and re-add with new priority
+            temp_items = []
+            found = False
+
+            while not self._queue.empty():
+                try:
+                    item = self._queue.get_nowait()
+                    if item.book_id == task_id:  # QueueItem uses book_id as the ID field
+                        # Create new item with updated priority
+                        new_item = QueueItem(task_id, new_priority, item.added_time)
+                        temp_items.append(new_item)
+                        found = True
+                        # Update task data priority
+                        if task_id in self._task_data:
+                            self._task_data[task_id].priority = new_priority
+                    else:
+                        temp_items.append(item)
+                except queue.Empty:
+                    break
+
+            # Put all items back
+            for item in temp_items:
+                self._queue.put(item)
+
+            return found
+
+    def reorder_queue(self, task_priorities: Dict[str, int]) -> bool:
+        """Bulk reorder queue by mapping task_id to new priority."""
+        with self._lock:
+            # Extract all items from queue
+            all_items = []
+            while not self._queue.empty():
+                try:
+                    item = self._queue.get_nowait()
+                    task_id = item.book_id  # QueueItem uses book_id as the ID field
+                    # Update priority if specified
+                    if task_id in task_priorities:
+                        new_priority = task_priorities[task_id]
+                        item = QueueItem(task_id, new_priority, item.added_time)
+                        # Update task data priority
+                        if task_id in self._task_data:
+                            self._task_data[task_id].priority = new_priority
+                    all_items.append(item)
+                except queue.Empty:
+                    break
+
+            # Put all items back with updated priorities
+            for item in all_items:
+                self._queue.put(item)
+
+            return True
+
+    def get_active_downloads(self) -> List[str]:
+        """Get list of currently active download task IDs."""
+        with self._lock:
+            return list(self._active_downloads.keys())
+
+    def has_pending_work(self) -> bool:
+        """Check if there are any active downloads or queued items."""
+        with self._lock:
+            if self._active_downloads:
+                return True
+            return any(status == QueueStatus.QUEUED for status in self._status.values())
+
+    def clear_completed(self) -> int:
+        """Remove all completed, errored, or cancelled tasks from tracking."""
+        terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED}
+        with self._lock:
+            to_remove = [task_id for task_id, status in self._status.items() if status in terminal_statuses]
+
+            for task_id in to_remove:
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
+                self._cancel_flags.pop(task_id, None)
+                self._active_downloads.pop(task_id, None)
+
+            return len(to_remove)
+
+    def refresh(self) -> None:
+        """Remove any tasks that are done downloading or have stale status."""
+        terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED}
+        with self._lock:
+            current_time = datetime.now()
+            to_remove = []
+
+            for task_id, status in self._status.items():
+                task = self._task_data.get(task_id)
+                if not task:
+                    continue
+
+                # Clear stale download paths
+                if task.download_path and not Path(task.download_path).exists():
+                    task.download_path = None
+
+                # Mark available downloads as done if file is gone
+                if status == QueueStatus.AVAILABLE and not task.download_path:
+                    self._update_status(task_id, QueueStatus.DONE)
+
+                # Check for stale status entries
+                last_update = self._status_timestamps.get(task_id)
+                if last_update and (current_time - last_update) > self._status_timeout:
+                    if status in terminal_statuses:
+                        to_remove.append(task_id)
+
+            # Remove stale entries
+            for task_id in to_remove:
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
+
+# Global instance of BookQueue
+book_queue = BookQueue()

shelfmark/core/search_plan.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional
+
+MANUAL_QUERY_MAX_LEN = 256
+
+from shelfmark.core.config import config
+from shelfmark.metadata_providers import (
+    BookMetadata,
+    group_languages_by_localized_title,
+    build_localized_search_titles,
+)
+
+
+@dataclass(frozen=True)
+class ReleaseSearchVariant:
+    """A single search variant (title + author) associated with languages."""
+
+    title: str
+    author: str
+    languages: Optional[List[str]] = None
+
+    @property
+    def query(self) -> str:
+        return " ".join(part for part in [self.title, self.author] if part).strip()
+
+
+@dataclass(frozen=True)
+class ReleaseSearchPlan:
+    """Pre-computed search inputs shared across release sources."""
+
+    languages: Optional[List[str]]
+    isbn_candidates: List[str]
+    author: str
+    title_variants: List[ReleaseSearchVariant]
+    grouped_title_variants: List[ReleaseSearchVariant]
+    manual_query: Optional[str] = None
+
+    @property
+    def primary_query(self) -> str:
+        return self.title_variants[0].query if self.title_variants else ""
+
+
+def _normalize_languages(languages: Optional[List[str]]) -> Optional[List[str]]:
+    if not languages:
+        default = config.BOOK_LANGUAGE
+        if not default:
+            return None
+        return [str(lang).strip() for lang in default if str(lang).strip()]
+
+    normalized: List[str] = []
+    for lang in languages:
+        if not lang:
+            continue
+        s = str(lang).strip()
+        if not s:
+            continue
+        normalized.append(s)
+
+    if any(lang.lower() == "all" for lang in normalized):
+        return None
+
+    return normalized or None
+
+
+def _pick_search_author(book: BookMetadata) -> str:
+    if book.search_author:
+        return book.search_author
+
+    if not book.authors:
+        return ""
+
+    first = book.authors[0]
+    if "," in first:
+        first = first.split(",")[0].strip()
+
+    return first
+
+
+def _pick_search_title(book: BookMetadata) -> str:
+    return book.search_title or book.title
+
+
+def build_release_search_plan(
+    book: BookMetadata,
+    languages: Optional[List[str]] = None,
+    manual_query: Optional[str] = None,
+) -> ReleaseSearchPlan:
+    resolved_languages = _normalize_languages(languages)
+
+    resolved_manual_query = None
+    if manual_query:
+        resolved_manual_query = manual_query.strip()[:MANUAL_QUERY_MAX_LEN] or None
+
+    author = _pick_search_author(book)
+    base_title = _pick_search_title(book)
+
+    if resolved_manual_query:
+        # Manual override: use the raw query as-is (no language/title expansion).
+        variant = ReleaseSearchVariant(title=resolved_manual_query, author="", languages=None)
+        return ReleaseSearchPlan(
+            languages=resolved_languages,
+            isbn_candidates=[],
+            author="",
+            title_variants=[variant],
+            grouped_title_variants=[variant],
+            manual_query=resolved_manual_query,
+        )
+
+    isbn_candidates: List[str] = []
+    if book.isbn_13:
+        isbn_candidates.append(book.isbn_13)
+    if book.isbn_10 and book.isbn_10 not in isbn_candidates:
+        isbn_candidates.append(book.isbn_10)
+
+    titles_by_language = book.titles_by_language or None
+    if book.search_title and titles_by_language:
+        titles_by_language = {
+            k: v
+            for k, v in titles_by_language.items()
+            if str(k).strip().lower() not in {"en", "eng", "english"}
+        }
+
+    grouped = group_languages_by_localized_title(
+        base_title=base_title,
+        languages=resolved_languages,
+        titles_by_language=titles_by_language,
+    )
+
+    grouped_variants: List[ReleaseSearchVariant] = [
+        ReleaseSearchVariant(title=title, author=author, languages=langs)
+        for title, langs in grouped
+        if title
+    ]
+
+    expanded_titles = build_localized_search_titles(
+        base_title=base_title,
+        languages=resolved_languages,
+        titles_by_language=titles_by_language,
+        excluded_languages={"en", "eng", "english"},
+    )
+
+    title_variants: List[ReleaseSearchVariant] = [
+        ReleaseSearchVariant(title=title, author=author, languages=None)
+        for title in expanded_titles
+        if title
+    ]
+
+    # If no titles could be built, fall back to ISBN queries.
+    if not title_variants and isbn_candidates:
+        title_variants = [
+            ReleaseSearchVariant(title=isbn, author="", languages=None)
+            for isbn in isbn_candidates
+        ]
+
+    return ReleaseSearchPlan(
+        languages=resolved_languages,
+        isbn_candidates=isbn_candidates,
+        author=author,
+        title_variants=title_variants,
+        grouped_title_variants=grouped_variants,
+        manual_query=None,
+    )

shelfmark/core/settings_registry.py

@@ -0,0 +1,906 @@
+"""Plugin settings registry with config file persistence."""
+
+import json
+import os
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Type, Union
+from threading import Lock
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@dataclass
+class FieldBase:
+    """Base class for all settings fields."""
+    key: str                              # Environment variable / config key
+    label: str                            # Display label in UI
+    description: str = ""                 # Help text
+    default: Any = None                   # Default value if not set
+    required: bool = False                # Whether field must have a value
+    env_var: Optional[str] = None         # Override env var name (defaults to key)
+    env_supported: bool = True            # Whether this setting can be set via ENV var (False = UI-only)
+    disabled: bool = False                # Whether field is disabled/greyed out
+    disabled_reason: str = ""             # Explanation shown when disabled
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    disabled_when: Optional[Dict[str, Any]] = None  # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
+    requires_restart: bool = False        # Whether changing this setting requires a container restart
+    universal_only: bool = False          # Only show in Universal search mode (hide in Direct mode)
+
+    def get_env_var_name(self) -> str:
+        """Get the environment variable name for this field."""
+        return self.env_var or self.key
+
+    def get_field_type(self) -> str:
+        """Get the field type name for serialization."""
+        return self.__class__.__name__
+
+
+@dataclass
+class TextField(FieldBase):
+    """Single-line text input."""
+    placeholder: str = ""
+    max_length: Optional[int] = None
+
+
+@dataclass
+class PasswordField(FieldBase):
+    """Password input (masked in UI, not returned in API responses)."""
+    placeholder: str = ""
+
+
+@dataclass
+class NumberField(FieldBase):
+    """Numeric input."""
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+    step: float = 1
+    default: float = 0
+
+
+@dataclass
+class CheckboxField(FieldBase):
+    """Boolean checkbox."""
+    default: bool = False
+
+
+@dataclass
+class SelectField(FieldBase):
+    """Single-choice dropdown."""
+    # Options can be a list or a callable that returns a list (for lazy evaluation)
+    options: Any = field(default_factory=list)  # [{value: "", label: ""}] or callable
+    filter_by_field: Optional[str] = None  # Field key whose value filters options via childOf property
+
+
+@dataclass
+class MultiSelectField(FieldBase):
+    """Multiple-choice selection."""
+    # Options can be a list or a callable that returns a list (for lazy evaluation)
+    options: Any = field(default_factory=list)  # [{value: "", label: ""}] or callable
+    default: List[str] = field(default_factory=list)
+    variant: str = "pills"  # "pills" (default) or "dropdown" for checkbox dropdown style
+
+
+@dataclass
+class OrderableListField(FieldBase):
+    # Options can be a list or a callable that returns a list (for lazy evaluation)
+    # Each option: {id, label, description?, disabledReason?, isLocked?, section?, isPinned?}
+    # - isLocked: toggle is disabled (can't enable/disable)
+    # - isPinned: can't be reordered (but toggle may still work if not also isLocked)
+    options: Any = field(default_factory=list)
+    # Default value: [{id, enabled}, ...] in priority order
+    default: List[Dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass
+class TableField(FieldBase):
+    """Editable table of structured rows."""
+
+    # Column definitions: [{key, label, type, placeholder?, options?, defaultValue?}, ...]
+    columns: Any = field(default_factory=list)  # list or callable
+
+    # Value format: list of objects
+    default: List[Dict[str, Any]] = field(default_factory=list)
+
+    add_label: str = "Add"
+    empty_message: str = ""
+
+
+@dataclass
+class ActionButton:
+    key: str                              # Action identifier
+    label: str                            # Button text
+    description: str = ""                 # Help text
+    style: str = "default"                # "default", "primary", "danger"
+    callback: Optional[Callable[..., Dict[str, Any]]] = None  # Returns {"success": bool, "message": str}
+    disabled: bool = False                # Whether button is disabled/greyed out
+    disabled_reason: str = ""             # Explanation shown when disabled
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    disabled_when: Optional[Dict[str, Any]] = None  # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
+
+    def get_field_type(self) -> str:
+        return "ActionButton"
+
+
+@dataclass
+class HeadingField:
+    """
+    Display-only heading with title and description.
+
+    Used to add section titles and descriptive text to settings pages.
+    Not an input field - purely for display.
+    """
+    key: str                              # Unique identifier
+    title: str                            # Heading title
+    description: str = ""                 # Description text (supports markdown-style links)
+    link_url: str = ""                    # Optional URL for a link
+    link_text: str = ""                   # Text for the link (defaults to URL if not provided)
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    universal_only: bool = False          # Only show in Universal search mode (hide in Direct mode)
+
+    def get_field_type(self) -> str:
+        return "HeadingField"
+
+
+# Type alias for all field types
+SettingsField = Union[TextField, PasswordField, NumberField, CheckboxField, SelectField, MultiSelectField, OrderableListField, ActionButton, HeadingField]
+
+
+@dataclass
+class SettingsTab:
+    """A tab/section in the settings UI."""
+    name: str                             # Internal name (used in URLs)
+    display_name: str                     # Display name in UI
+    fields: List[SettingsField] = field(default_factory=list)
+    icon: Optional[str] = None            # Icon name for UI
+    order: int = 100                      # Sort order (lower = earlier)
+    group: Optional[str] = None           # Group name this tab belongs to
+
+
+@dataclass
+class SettingsGroup:
+    """A collapsible group of settings tabs in the UI."""
+    name: str                             # Internal name
+    display_name: str                     # Display name in UI
+    icon: Optional[str] = None            # Icon name for UI
+    order: int = 100                      # Sort order (lower = earlier)
+
+
+_SETTINGS_REGISTRY: Dict[str, SettingsTab] = {}
+_GROUPS_REGISTRY: Dict[str, SettingsGroup] = {}
+_ON_SAVE_HANDLERS: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {}
+_REGISTRY_LOCK = Lock()
+
+
+def register_group(
+    name: str,
+    display_name: str,
+    icon: Optional[str] = None,
+    order: int = 100
+) -> None:
+    with _REGISTRY_LOCK:
+        group = SettingsGroup(
+            name=name,
+            display_name=display_name,
+            icon=icon,
+            order=order,
+        )
+        _GROUPS_REGISTRY[name] = group
+        logger.debug(f"Registered settings group: {name}")
+
+
+def register_settings(
+    name: str,
+    display_name: str,
+    icon: Optional[str] = None,
+    order: int = 100,
+    group: Optional[str] = None
+):
+    def decorator(func: Callable[[], List[SettingsField]]):
+        with _REGISTRY_LOCK:
+            fields = func()
+            tab = SettingsTab(
+                name=name,
+                display_name=display_name,
+                fields=fields,
+                icon=icon,
+                order=order,
+                group=group,
+            )
+            _SETTINGS_REGISTRY[name] = tab
+            logger.debug(f"Registered settings tab: {name} ({len(fields)} fields)" +
+                        (f" in group {group}" if group else ""))
+        return func
+    return decorator
+
+
+def register_on_save(
+    tab_name: str,
+    handler: Callable[[Dict[str, Any]], Dict[str, Any]]
+) -> None:
+    with _REGISTRY_LOCK:
+        _ON_SAVE_HANDLERS[tab_name] = handler
+        logger.debug(f"Registered on_save handler for tab: {tab_name}")
+
+
+def get_on_save_handler(tab_name: str) -> Optional[Callable[[Dict[str, Any]], Dict[str, Any]]]:
+    """Get the on_save handler for a settings tab, if any."""
+    return _ON_SAVE_HANDLERS.get(tab_name)
+
+
+def get_settings_tab(name: str) -> Optional[SettingsTab]:
+    """Get a specific settings tab by name."""
+    return _SETTINGS_REGISTRY.get(name)
+
+
+def get_all_settings_tabs() -> List[SettingsTab]:
+    """Get all registered settings tabs, sorted by order."""
+    return sorted(_SETTINGS_REGISTRY.values(), key=lambda t: (t.order, t.name))
+
+
+def list_registered_settings() -> List[str]:
+    """List all registered settings tab names."""
+    return list(_SETTINGS_REGISTRY.keys())
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def _get_config_file_path(tab_name: str) -> Path:
+    """Get the config file path for a settings tab."""
+    config_dir = _get_config_dir()
+    # Core settings tabs share the main settings.json file
+    if tab_name in ("general", "search_mode"):
+        return config_dir / "settings.json"
+    return config_dir / "plugins" / f"{tab_name}.json"
+
+
+def _ensure_config_dir(tab_name: str) -> None:
+    """Ensure the config directory exists."""
+    config_path = _get_config_file_path(tab_name)
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def load_config_file(tab_name: str) -> Dict[str, Any]:
+    config_path = _get_config_file_path(tab_name)
+
+    if not config_path.exists():
+        return {}
+
+    try:
+        with open(config_path, 'r') as f:
+            return json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(f"Invalid JSON in config file {config_path}: {e}")
+        return {}
+
+
+def save_config_file(tab_name: str, values: Dict[str, Any]) -> bool:
+    try:
+        _ensure_config_dir(tab_name)
+        config_path = _get_config_file_path(tab_name)
+
+        # Load existing config and merge
+        existing = load_config_file(tab_name)
+        existing.update(values)
+
+        with open(config_path, 'w') as f:
+            json.dump(existing, f, indent=2)
+
+        logger.info(f"Saved settings to {config_path}")
+        return True
+    except Exception as e:
+        logger.error(f"Error saving config file for {tab_name}: {e}")
+        return False
+
+
+def initialize_default_configs() -> bool:
+    """Initialize config files with default values on first startup.
+
+    Creates config files for all settings tabs that don't have one yet,
+    populating them with field default values. This ensures config files
+    exist from first startup rather than only being created on explicit save.
+
+    Returns:
+        True if initialization succeeded or was skipped (already initialized),
+        False if there was an error accessing the config directory.
+    """
+    try:
+        config_dir = _get_config_dir()
+
+        # Check if config directory exists and is writable
+        if not config_dir.exists():
+            logger.warning(f"Config directory does not exist: {config_dir}")
+            return False
+
+        # Test writability
+        test_file = config_dir / ".write_test"
+        try:
+            test_file.touch()
+            test_file.unlink()
+        except (OSError, PermissionError) as e:
+            logger.warning(f"Config directory is not writable: {config_dir} - {e}")
+            return False
+
+        initialized_tabs = []
+
+        for tab in get_all_settings_tabs():
+            config_path = _get_config_file_path(tab.name)
+
+            # Skip if config file already exists
+            if config_path.exists():
+                continue
+
+            # Collect default values for all fields
+            defaults = {}
+            for field in tab.fields:
+                # Skip non-value fields
+                if isinstance(field, (ActionButton, HeadingField)):
+                    continue
+
+                # Only include fields that have a non-None default
+                if field.default is not None:
+                    defaults[field.key] = field.default
+
+            # Create config file with defaults if we have any
+            if defaults:
+                _ensure_config_dir(tab.name)
+                try:
+                    with open(config_path, 'w') as f:
+                        json.dump(defaults, f, indent=2)
+                    initialized_tabs.append(tab.name)
+                except Exception as e:
+                    logger.error(f"Failed to initialize config for {tab.name}: {e}")
+
+        if initialized_tabs:
+            logger.info(f"Initialized default configs for: {initialized_tabs}")
+
+        return True
+
+    except Exception as e:
+        logger.error(f"Error during config initialization: {e}")
+        return False
+
+
+def sync_env_to_config() -> None:
+    # Initialize default configs first (for fresh installs)
+    initialize_default_configs()
+
+    for tab in get_all_settings_tabs():
+        values_to_sync = {}
+
+        for field in tab.fields:
+            # Skip non-value fields
+            if isinstance(field, (ActionButton, HeadingField)):
+                continue
+
+            # Skip fields that don't support ENV vars
+            if not getattr(field, 'env_supported', True):
+                continue
+
+            # Check if ENV var is set
+            env_var_name = field.get_env_var_name()
+            env_value = os.environ.get(env_var_name)
+
+            if env_value is not None:
+                # Parse the ENV value to the appropriate type
+                parsed_value = _parse_env_value(env_value, field)
+                values_to_sync[field.key] = parsed_value
+
+        # Save synced values to config file (merge with existing)
+        if values_to_sync:
+            save_config_file(tab.name, values_to_sync)
+            logger.debug(f"Synced {len(values_to_sync)} ENV values to {tab.name} config: {list(values_to_sync.keys())}")
+
+    migrate_legacy_settings()
+
+
+def migrate_legacy_settings() -> None:
+    """Migrate legacy settings to new unified file destination format.
+
+    Maps old settings to new:
+    - PROCESSING_MODE + USE_BOOK_TITLE -> FILE_ORGANIZATION
+    - INGEST_DIR / LIBRARY_PATH -> DESTINATION
+    - LIBRARY_TEMPLATE -> TEMPLATE
+    - USE_CONTENT_TYPE_DIRECTORIES -> AA_CONTENT_TYPE_ROUTING
+    - INGEST_DIR_* -> AA_CONTENT_TYPE_DIR_*
+    - TORRENT_HARDLINK -> HARDLINK_TORRENTS / HARDLINK_TORRENTS_AUDIOBOOK
+    """
+    # Load existing downloads config
+    downloads_config = load_config_file("downloads")
+    source_config = load_config_file("download_sources")
+
+    # Skip migration if already using new settings
+    if "FILE_ORGANIZATION" in downloads_config or "DESTINATION" in downloads_config:
+        return
+
+    # Skip migration if no legacy settings exist (fresh install)
+    legacy_keys = {
+        "PROCESSING_MODE", "INGEST_DIR", "LIBRARY_PATH", "USE_BOOK_TITLE",
+        "LIBRARY_TEMPLATE", "PROCESSING_MODE_AUDIOBOOK", "INGEST_DIR_AUDIOBOOK",
+        "LIBRARY_PATH_AUDIOBOOK", "LIBRARY_TEMPLATE_AUDIOBOOK", "TORRENT_HARDLINK",
+        "USE_CONTENT_TYPE_DIRECTORIES",
+    }
+    if not any(key in downloads_config for key in legacy_keys):
+        return
+
+    migrated_downloads = {}
+    migrated_sources = {}
+
+    # === BOOKS MIGRATION ===
+    old_mode = downloads_config.get("PROCESSING_MODE", "ingest")
+    old_ingest_dir = downloads_config.get("INGEST_DIR", "/cwa-book-ingest")
+    old_library_path = downloads_config.get("LIBRARY_PATH", "")
+    old_use_book_title = downloads_config.get("USE_BOOK_TITLE", True)
+    old_library_template = downloads_config.get("LIBRARY_TEMPLATE", "{Author}/{Title}")
+
+    # Map PROCESSING_MODE + USE_BOOK_TITLE -> FILE_ORGANIZATION
+    if old_mode == "library":
+        migrated_downloads["FILE_ORGANIZATION"] = "organize"
+        migrated_downloads["DESTINATION"] = old_library_path or "/books"
+        migrated_downloads["TEMPLATE"] = old_library_template
+    else:
+        if old_use_book_title:
+            migrated_downloads["FILE_ORGANIZATION"] = "rename"
+            migrated_downloads["TEMPLATE"] = "{Author} - {Title} ({Year})"
+        else:
+            migrated_downloads["FILE_ORGANIZATION"] = "none"
+        migrated_downloads["DESTINATION"] = old_ingest_dir
+
+    # === AUDIOBOOKS MIGRATION ===
+    old_mode_ab = downloads_config.get("PROCESSING_MODE_AUDIOBOOK", "ingest")
+    old_ingest_dir_ab = downloads_config.get("INGEST_DIR_AUDIOBOOK", "")
+    old_library_path_ab = downloads_config.get("LIBRARY_PATH_AUDIOBOOK", "")
+    old_library_template_ab = downloads_config.get("LIBRARY_TEMPLATE_AUDIOBOOK", "{Author}/{Title}")
+
+    if old_mode_ab == "library":
+        migrated_downloads["FILE_ORGANIZATION_AUDIOBOOK"] = "organize"
+        migrated_downloads["DESTINATION_AUDIOBOOK"] = old_library_path_ab or ""
+        migrated_downloads["TEMPLATE_AUDIOBOOK"] = old_library_template_ab
+    else:
+        migrated_downloads["FILE_ORGANIZATION_AUDIOBOOK"] = "rename"
+        migrated_downloads["TEMPLATE_AUDIOBOOK"] = "{Author} - {Title}"
+        if old_ingest_dir_ab:
+            migrated_downloads["DESTINATION_AUDIOBOOK"] = old_ingest_dir_ab
+
+    # === HARDLINK MIGRATION ===
+    old_torrent_hardlink = downloads_config.get("TORRENT_HARDLINK")
+    if old_torrent_hardlink is not None:
+        # Books default to False (ingest folder use case)
+        # Audiobooks default to True (library folder use case)
+        # But if explicitly set, apply to both
+        migrated_downloads["HARDLINK_TORRENTS"] = old_torrent_hardlink
+        migrated_downloads["HARDLINK_TORRENTS_AUDIOBOOK"] = old_torrent_hardlink
+
+    # === CONTENT-TYPE ROUTING MIGRATION ===
+    old_use_content_type = downloads_config.get("USE_CONTENT_TYPE_DIRECTORIES", False)
+    if old_use_content_type:
+        migrated_sources["AA_CONTENT_TYPE_ROUTING"] = True
+
+        # Map old keys to new keys
+        content_type_mapping = {
+            "INGEST_DIR_BOOK_FICTION": "AA_CONTENT_TYPE_DIR_FICTION",
+            "INGEST_DIR_BOOK_NON_FICTION": "AA_CONTENT_TYPE_DIR_NON_FICTION",
+            "INGEST_DIR_BOOK_UNKNOWN": "AA_CONTENT_TYPE_DIR_UNKNOWN",
+            "INGEST_DIR_MAGAZINE": "AA_CONTENT_TYPE_DIR_MAGAZINE",
+            "INGEST_DIR_COMIC_BOOK": "AA_CONTENT_TYPE_DIR_COMIC",
+            "INGEST_DIR_STANDARDS_DOCUMENT": "AA_CONTENT_TYPE_DIR_STANDARDS",
+            "INGEST_DIR_MUSICAL_SCORE": "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+            "INGEST_DIR_OTHER": "AA_CONTENT_TYPE_DIR_OTHER",
+        }
+
+        for old_key, new_key in content_type_mapping.items():
+            old_value = downloads_config.get(old_key, "")
+            if old_value:
+                migrated_sources[new_key] = old_value
+
+    # Save migrated settings
+    if migrated_downloads:
+        save_config_file("downloads", migrated_downloads)
+        logger.info(f"Migrated download settings: {list(migrated_downloads.keys())}")
+
+    if migrated_sources:
+        save_config_file("download_sources", migrated_sources)
+        logger.info(f"Migrated content-type routing settings: {list(migrated_sources.keys())}")
+
+
+def get_setting_value(field: SettingsField, tab_name: str) -> Any:
+    if isinstance(field, (ActionButton, HeadingField)):
+        return None  # Actions and headings don't have values
+
+    # 1. Check environment variable (if supported for this field)
+    if field.env_supported:
+        env_var_name = field.get_env_var_name()
+        env_value = os.environ.get(env_var_name)
+        if env_value is not None:
+            return _parse_env_value(env_value, field)
+
+    # 2. Check config file
+    config = load_config_file(tab_name)
+    if field.key in config:
+        return config[field.key]
+
+    # 3. Return default
+    return field.default
+
+
+def _parse_env_value(value: str, field: SettingsField) -> Any:
+    """Parse an environment variable value to the appropriate type."""
+    if isinstance(field, CheckboxField):
+        return value.lower() in ('true', '1', 'yes', 'on')
+    elif isinstance(field, NumberField):
+        try:
+            if '.' in value:
+                return float(value)
+            return int(value)
+        except ValueError:
+            return field.default
+    elif isinstance(field, MultiSelectField):
+        return [v.strip() for v in value.split(',') if v.strip()]
+    elif isinstance(field, OrderableListField):
+        # Parse JSON array: [{"id": "...", "enabled": true}, ...]
+        try:
+            return json.loads(value)
+        except json.JSONDecodeError:
+            logger.warning(f"Invalid JSON for {field.key}, using default")
+            return field.default
+    elif isinstance(field, TableField):
+        # Parse JSON array: [{"col": "value"}, ...]
+        try:
+            parsed = json.loads(value)
+            return parsed if isinstance(parsed, list) else field.default
+        except json.JSONDecodeError:
+            logger.warning(f"Invalid JSON for {field.key}, using default")
+            return field.default
+    else:
+        return value
+
+
+def is_value_from_env(field: SettingsField) -> bool:
+    """Check if a field's value comes from an environment variable."""
+    if isinstance(field, (ActionButton, HeadingField)):
+        return False
+    # UI-only settings never come from ENV (env_supported=False)
+    if not getattr(field, 'env_supported', True):
+        return False
+    return field.get_env_var_name() in os.environ
+
+
+def serialize_field(field: SettingsField, tab_name: str, include_value: bool = True) -> Dict[str, Any]:
+    """
+    Serialize a field for API response.
+
+    Args:
+        field: The settings field.
+        tab_name: The settings tab name.
+        include_value: Whether to include the current value.
+
+    Returns:
+        Dict representation of the field.
+    """
+    # HeadingField has a different structure - handle separately
+    if isinstance(field, HeadingField):
+        result: Dict[str, Any] = {
+            "key": field.key,
+            "type": field.get_field_type(),
+            "title": field.title,
+            "description": field.description,
+        }
+        if field.link_url:
+            result["linkUrl"] = field.link_url
+            result["linkText"] = field.link_text or field.link_url
+        if field.show_when:
+            result["showWhen"] = field.show_when
+        if field.universal_only:
+            result["universalOnly"] = True
+        return result
+
+    result: Dict[str, Any] = {
+        "key": field.key,
+        "label": field.label,
+        "type": field.get_field_type(),
+        "description": getattr(field, 'description', ''),
+        "required": getattr(field, 'required', False),
+        "disabled": getattr(field, 'disabled', False),
+        "disabledReason": getattr(field, 'disabled_reason', ''),
+        "requiresRestart": getattr(field, 'requires_restart', False),
+    }
+
+    # Add optional properties if set
+    if getattr(field, 'show_when', None):
+        result["showWhen"] = field.show_when
+    if getattr(field, 'disabled_when', None):
+        result["disabledWhen"] = field.disabled_when
+    if getattr(field, 'universal_only', False):
+        result["universalOnly"] = True
+
+    # Add type-specific properties
+    if isinstance(field, TextField):
+        result["placeholder"] = field.placeholder
+        if field.max_length:
+            result["maxLength"] = field.max_length
+    elif isinstance(field, PasswordField):
+        result["placeholder"] = field.placeholder
+    elif isinstance(field, NumberField):
+        result["min"] = field.min_value
+        result["max"] = field.max_value
+        result["step"] = field.step
+    elif isinstance(field, SelectField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+        if field.default is not None:
+            result["default"] = field.default
+        if field.filter_by_field:
+            result["filterByField"] = field.filter_by_field
+    elif isinstance(field, MultiSelectField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+        result["variant"] = field.variant
+    elif isinstance(field, OrderableListField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+    elif isinstance(field, TableField):
+        columns = field.columns() if callable(field.columns) else field.columns
+        result["columns"] = columns
+        result["addLabel"] = field.add_label
+        result["emptyMessage"] = field.empty_message
+    elif isinstance(field, ActionButton):
+        result["style"] = field.style
+        result["description"] = field.description
+
+    if include_value and not isinstance(field, (ActionButton, HeadingField)):
+        value = get_setting_value(field, tab_name)
+
+        # Ensure select values are serialized as strings so the frontend can
+        # reliably match against string option values.
+        if isinstance(field, SelectField) and value is not None:
+            value = str(value)
+        elif isinstance(field, MultiSelectField):
+            if value is None:
+                value = []
+            elif isinstance(value, list):
+                value = [str(v) for v in value]
+            elif isinstance(value, str):
+                # Support legacy/manual configs where MultiSelect values were saved
+                # as comma-separated strings.
+                value = [v.strip() for v in value.split(",") if v.strip()]
+            else:
+                value = []
+        elif isinstance(field, TableField):
+            if value is None:
+                value = []
+            elif not isinstance(value, list):
+                value = []
+
+        result["value"] = value if value is not None else ""
+        result["fromEnv"] = is_value_from_env(field)
+
+    return result
+
+
+def serialize_tab(tab: SettingsTab, include_values: bool = True) -> Dict[str, Any]:
+    """Serialize a settings tab for API response."""
+    return {
+        "name": tab.name,
+        "displayName": tab.display_name,
+        "icon": tab.icon,
+        "order": tab.order,
+        "group": tab.group,
+        "fields": [serialize_field(f, tab.name, include_values) for f in tab.fields],
+    }
+
+
+def serialize_group(group: SettingsGroup) -> Dict[str, Any]:
+    """Serialize a settings group for API response."""
+    return {
+        "name": group.name,
+        "displayName": group.display_name,
+        "icon": group.icon,
+        "order": group.order,
+    }
+
+
+def get_all_groups() -> List[SettingsGroup]:
+    """Get all registered settings groups, sorted by order."""
+    return sorted(_GROUPS_REGISTRY.values(), key=lambda g: (g.order, g.name))
+
+
+def serialize_all_settings(include_values: bool = True) -> Dict[str, Any]:
+    """Serialize all settings for API response."""
+    tabs = get_all_settings_tabs()
+    groups = get_all_groups()
+    return {
+        "tabs": [serialize_tab(t, include_values) for t in tabs],
+        "groups": [serialize_group(g) for g in groups],
+    }
+
+
+def execute_action(tab_name: str, action_key: str, current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """
+    Execute an action button's callback.
+
+    Args:
+        tab_name: The settings tab name.
+        action_key: The action key to execute.
+        current_values: Optional dict of current form values (unsaved).
+                       Passed to callbacks that accept it.
+
+    Returns:
+        Dict with "success" (bool) and "message" (str).
+    """
+    import inspect
+
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        return {"success": False, "message": f"Unknown settings tab: {tab_name}"}
+
+    for field in tab.fields:
+        if isinstance(field, ActionButton) and field.key == action_key:
+            if field.callback:
+                try:
+                    # Check if callback accepts current_values parameter
+                    sig = inspect.signature(field.callback)
+                    if "current_values" in sig.parameters:
+                        return field.callback(current_values=current_values or {})
+                    else:
+                        return field.callback()
+                except Exception as e:
+                    logger.error(f"Action {action_key} failed: {e}")
+                    return {"success": False, "message": str(e)}
+            else:
+                return {"success": False, "message": "Action has no callback defined"}
+
+    return {"success": False, "message": f"Unknown action: {action_key}"}
+
+
+def _sync_metadata_provider_selection() -> None:
+    """
+    Sync the METADATA_PROVIDER setting based on enabled providers.
+
+    Called after saving metadata provider settings to auto-select
+    the first enabled provider if the current selection is invalid.
+    """
+    try:
+        from shelfmark.metadata_providers import sync_metadata_provider_selection
+        sync_metadata_provider_selection()
+    except ImportError:
+        pass  # Metadata providers module not available
+
+
+def _apply_dns_settings(config) -> None:
+    """
+    Apply DNS settings changes to the network module.
+
+    This ensures DNS changes take effect immediately without requiring
+    a container restart.
+    """
+    try:
+        from shelfmark.download import network
+
+        provider = config.get("CUSTOM_DNS", "auto")
+        use_doh = config.get("USE_DOH", False)
+        manual_servers = None
+
+        if provider == "manual":
+            manual_dns = config.get("CUSTOM_DNS_MANUAL", "")
+            if manual_dns:
+                # Parse comma-separated server list
+                manual_servers = [s.strip() for s in manual_dns.split(",") if s.strip()]
+
+        network.set_dns_provider(provider, manual_servers, use_doh=use_doh)
+    except ImportError:
+        pass  # Network module not available
+    except Exception as e:
+        logger.warning(f"Failed to apply DNS settings: {e}")
+
+
+def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]:
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        return {"success": False, "message": f"Unknown settings tab: {tab_name}", "updated": [], "requiresRestart": False}
+
+    # Build a map of field keys to fields (exclude non-value fields)
+    field_map = {f.key: f for f in tab.fields if not isinstance(f, (ActionButton, HeadingField))}
+
+    # Filter out values that are set via env vars or unknown
+    values_to_save = {}
+    skipped_env = []
+    skipped_unknown = []
+    restart_required_keys = []
+
+    for key, value in values.items():
+        if key not in field_map:
+            skipped_unknown.append(key)
+            continue
+
+        field = field_map[key]
+        if is_value_from_env(field):
+            skipped_env.append(key)
+            continue
+
+        # Handle password fields - only update if a new value is provided
+        if isinstance(field, PasswordField) and not value:
+            continue
+
+        values_to_save[key] = value
+
+        # Track if this field requires restart
+        if getattr(field, 'requires_restart', False):
+            restart_required_keys.append(key)
+
+    if not values_to_save:
+        message = "No settings to update"
+        if skipped_env:
+            message += f". Skipped (set via env): {', '.join(skipped_env)}"
+        return {"success": True, "message": message, "updated": [], "requiresRestart": False}
+
+    # Call on_save handler if registered (for custom validation/transformation)
+    on_save_handler = get_on_save_handler(tab_name)
+    if on_save_handler:
+        try:
+            result = on_save_handler(values_to_save.copy())
+            if result.get("error"):
+                return {
+                    "success": False,
+                    "message": result.get("message", "Validation failed"),
+                    "updated": [],
+                    "requiresRestart": False
+                }
+            # Use the transformed values
+            values_to_save = result.get("values", values_to_save)
+        except Exception as e:
+            logger.error(f"on_save handler for {tab_name} failed: {e}")
+            return {
+                "success": False,
+                "message": f"Save handler error: {str(e)}",
+                "updated": [],
+                "requiresRestart": False
+            }
+
+    # Save to config file
+    if save_config_file(tab_name, values_to_save):
+        # Refresh the config singleton so live settings take effect immediately
+        config_obj = None
+        try:
+            from shelfmark.core.config import config as config_obj
+
+            config_obj.refresh()
+        except ImportError:
+            config_obj = None  # Config module not yet available during initial setup
+
+        # Apply DNS settings changes live (network tab)
+        dns_keys = {"CUSTOM_DNS", "CUSTOM_DNS_MANUAL", "USE_DOH"}
+        if (
+            config_obj is not None
+            and tab_name == "network"
+            and dns_keys.intersection(values_to_save.keys())
+        ):
+            _apply_dns_settings(config_obj)
+
+        # Sync metadata provider selection when a provider's enabled state changes
+        tab = get_settings_tab(tab_name)
+        if tab and tab.group == "metadata_providers":
+            _sync_metadata_provider_selection()
+
+        message = f"Updated {len(values_to_save)} setting(s)"
+        if skipped_env:
+            message += f". Skipped (set via env): {', '.join(skipped_env)}"
+
+        requires_restart = len(restart_required_keys) > 0
+        return {
+            "success": True,
+            "message": message,
+            "updated": list(values_to_save.keys()),
+            "requiresRestart": requires_restart,
+            "restartRequiredFor": restart_required_keys,
+        }
+    else:
+        return {"success": False, "message": "Failed to save settings", "updated": [], "requiresRestart": False}

shelfmark/core/utils.py

@@ -0,0 +1,196 @@
+"""Shared utility functions for the Shelfmark."""
+
+import base64
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+
+def normalize_http_url(
+    url: Optional[str],
+    *,
+    default_scheme: str = "http",
+    strip_trailing_slash: bool = True,
+    allow_special: tuple[str, ...] = (),
+) -> str:
+    """Normalize a configured HTTP URL for requests and links."""
+    if not isinstance(url, str):
+        return ""
+
+    normalized = url.strip()
+    if not normalized:
+        return ""
+
+    if (normalized.startswith("\"") and normalized.endswith("\"")) or (
+        normalized.startswith("'") and normalized.endswith("'")
+    ):
+        normalized = normalized[1:-1].strip()
+        if not normalized:
+            return ""
+
+    if allow_special:
+        special_map = {
+            value.lower(): value
+            for value in allow_special
+            if isinstance(value, str)
+        }
+        special_match = special_map.get(normalized.lower())
+        if special_match is not None:
+            return special_match
+
+    if normalized.startswith(("/", "./", "../")):
+        return normalized
+
+    if "://" not in normalized:
+        scheme = default_scheme.strip().rstrip(":/")
+        if scheme:
+            normalized = f"{scheme}://{normalized}"
+
+    if strip_trailing_slash:
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def normalize_base_path(value: Optional[str]) -> str:
+    """Normalize a URL base path for reverse proxy subpath deployments."""
+    if not isinstance(value, str):
+        return ""
+
+    path = value.strip()
+    if not path:
+        return ""
+
+    if "://" in path:
+        parsed = urlparse(path)
+        path = parsed.path or ""
+
+    if not path or path == "/":
+        return ""
+
+    if not path.startswith("/"):
+        path = "/" + path
+
+    return path.rstrip("/")
+
+
+def is_audiobook(content_type: Optional[str]) -> bool:
+    """Check if content type indicates an audiobook."""
+    return bool(content_type and "audiobook" in content_type.lower())
+
+
+CONTENT_TYPES = [
+    "book (fiction)",
+    "book (non-fiction)",
+    "book (unknown)",
+    "magazine",
+    "comic book",
+    "audiobook",
+    "standards document",
+    "musical score",
+    "other",
+]
+
+# Maps AA content types to their config keys for content-type routing
+# Used when AA_CONTENT_TYPE_ROUTING is enabled
+_AA_CONTENT_TYPE_TO_CONFIG_KEY = {
+    "book (fiction)": "AA_CONTENT_TYPE_DIR_FICTION",
+    "book (non-fiction)": "AA_CONTENT_TYPE_DIR_NON_FICTION",
+    "book (unknown)": "AA_CONTENT_TYPE_DIR_UNKNOWN",
+    "magazine": "AA_CONTENT_TYPE_DIR_MAGAZINE",
+    "comic book": "AA_CONTENT_TYPE_DIR_COMIC",
+    "audiobook": "AA_CONTENT_TYPE_DIR_AUDIOBOOK",
+    "standards document": "AA_CONTENT_TYPE_DIR_STANDARDS",
+    "musical score": "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+    "other": "AA_CONTENT_TYPE_DIR_OTHER",
+}
+
+# Legacy mapping - kept for backwards compatibility during migration
+_LEGACY_CONTENT_TYPE_TO_CONFIG_KEY = {
+    "book (fiction)": "INGEST_DIR_BOOK_FICTION",
+    "book (non-fiction)": "INGEST_DIR_BOOK_NON_FICTION",
+    "book (unknown)": "INGEST_DIR_BOOK_UNKNOWN",
+    "magazine": "INGEST_DIR_MAGAZINE",
+    "comic book": "INGEST_DIR_COMIC_BOOK",
+    "audiobook": "INGEST_DIR_AUDIOBOOK",
+    "standards document": "INGEST_DIR_STANDARDS_DOCUMENT",
+    "musical score": "INGEST_DIR_MUSICAL_SCORE",
+    "other": "INGEST_DIR_OTHER",
+}
+
+
+def get_destination(is_audiobook: bool = False) -> Path:
+    """Get base destination directory. Audiobooks fall back to main destination."""
+    from shelfmark.core.config import config
+
+    if is_audiobook:
+        # Audiobook destination with fallback to main destination
+        audiobook_dest = config.get("DESTINATION_AUDIOBOOK", "")
+        if audiobook_dest:
+            return Path(audiobook_dest)
+
+    # Main destination (also fallback for audiobooks)
+    # Check new setting first, then legacy INGEST_DIR
+    destination = config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books")
+    return Path(destination)
+
+
+def get_aa_content_type_dir(content_type: Optional[str] = None) -> Optional[Path]:
+    """Get override directory for AA content-type routing if configured."""
+    from shelfmark.core.config import config
+
+    # Check if content-type routing is enabled (new or legacy setting)
+    if not config.get("AA_CONTENT_TYPE_ROUTING", False) and not config.get("USE_CONTENT_TYPE_DIRECTORIES", False):
+        return None
+
+    if not content_type:
+        return None
+
+    content_type_lower = content_type.lower().strip()
+
+    # Try new AA-specific config keys first, then legacy keys
+    for mapping in (_AA_CONTENT_TYPE_TO_CONFIG_KEY, _LEGACY_CONTENT_TYPE_TO_CONFIG_KEY):
+        config_key = mapping.get(content_type_lower)
+        if config_key:
+            custom_dir = config.get(config_key, "")
+            if custom_dir:
+                return Path(custom_dir)
+
+    return None
+
+
+def get_ingest_dir(content_type: Optional[str] = None) -> Path:
+    """DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead."""
+    from shelfmark.core.config import config
+
+    # Check new DESTINATION setting first, then legacy INGEST_DIR
+    default_ingest_dir = Path(config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books"))
+
+    if not content_type:
+        return default_ingest_dir
+
+    # Check for content-type override
+    override_dir = get_aa_content_type_dir(content_type)
+    if override_dir:
+        return override_dir
+
+    return default_ingest_dir
+
+
+def transform_cover_url(cover_url: Optional[str], cache_id: str) -> Optional[str]:
+    """Transform external cover URL to local proxy URL when caching is enabled."""
+    if not cover_url:
+        return cover_url
+
+    # Skip if already a local URL (starts with /)
+    if cover_url.startswith('/'):
+        return cover_url
+
+    # Check if cover caching is enabled
+    from shelfmark.config.env import is_covers_cache_enabled
+    if not is_covers_cache_enabled():
+        return cover_url
+
+    # Encode the original URL and create a proxy URL
+    encoded_url = base64.urlsafe_b64encode(cover_url.encode()).decode()
+    return f"/api/covers/{cache_id}?url={encoded_url}"

shelfmark/download/__init__.py

@@ -0,0 +1 @@
+"""Download module - HTTP downloads, network, and orchestration."""

shelfmark/download/archive.py

@@ -0,0 +1,239 @@
+"""Archive extraction utilities for downloaded book archives."""
+
+import os
+import shutil
+import zipfile
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.postprocess.policy import (
+    get_supported_audiobook_formats,
+    get_supported_formats,
+)
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.fs import atomic_write
+
+logger = setup_logger(__name__)
+
+
+# Check for rarfile availability at module load
+try:
+    import rarfile
+
+    RAR_AVAILABLE = True
+except ImportError:
+    RAR_AVAILABLE = False
+    logger.warning("rarfile not installed - RAR extraction disabled")
+
+
+class ArchiveExtractionError(Exception):
+    """Raised when archive extraction fails."""
+
+    pass
+
+
+class PasswordProtectedError(ArchiveExtractionError):
+    """Raised when archive requires a password."""
+
+    pass
+
+
+class CorruptedArchiveError(ArchiveExtractionError):
+    """Raised when archive is corrupted."""
+
+    pass
+
+
+def is_archive(file_path: Path) -> bool:
+    """Check if file is a supported archive format."""
+    suffix = file_path.suffix.lower().lstrip(".")
+    return suffix in ("zip", "rar")
+
+
+def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> bool:
+    """Check if file matches user's supported formats setting based on content type."""
+    ext = file_path.suffix.lower().lstrip(".")
+    if check_audiobook(content_type):
+        supported_formats = get_supported_audiobook_formats()
+    else:
+        supported_formats = get_supported_formats()
+    return ext in supported_formats
+
+
+# All known ebook extensions (superset of what user might enable)
+ALL_EBOOK_EXTENSIONS = {'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr', '.doc', '.docx', '.rtf', '.txt'}
+
+# All known audio extensions (superset of what user might enable for audiobooks)
+ALL_AUDIO_EXTENSIONS = {'.m4b', '.mp3', '.m4a', '.aac', '.flac', '.ogg', '.wma', '.wav', '.opus'}
+
+
+def _filter_files(
+    extracted_files: List[Path],
+    content_type: Optional[str] = None,
+) -> Tuple[List[Path], List[Path], List[Path]]:
+    """Filter files by content type. Returns (matched, rejected_format, other)."""
+    is_audiobook = check_audiobook(content_type)
+    known_extensions = ALL_AUDIO_EXTENSIONS if is_audiobook else ALL_EBOOK_EXTENSIONS
+
+    matched_files = []
+    rejected_format_files = []
+    other_files = []
+
+    for file_path in extracted_files:
+        if _is_supported_file(file_path, content_type):
+            matched_files.append(file_path)
+        elif file_path.suffix.lower() in known_extensions:
+            rejected_format_files.append(file_path)
+        else:
+            other_files.append(file_path)
+
+    return matched_files, rejected_format_files, other_files
+
+
+def extract_archive(
+    archive_path: Path,
+    output_dir: Path,
+    content_type: Optional[str] = None,
+) -> Tuple[List[Path], List[str], List[Path]]:
+    """Extract archive and filter by content type. Returns (matched, warnings, rejected)."""
+    suffix = archive_path.suffix.lower().lstrip(".")
+
+    if suffix == "zip":
+        extracted_files, warnings = _extract_zip(archive_path, output_dir)
+    elif suffix == "rar":
+        extracted_files, warnings = _extract_rar(archive_path, output_dir)
+    else:
+        raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
+
+    is_audiobook = check_audiobook(content_type)
+    file_type_label = "audiobook" if is_audiobook else "book"
+
+    # Filter files based on content type
+    matched_files, rejected_files, other_files = _filter_files(extracted_files, content_type)
+
+    # Delete rejected files (valid formats but not enabled by user)
+    for rejected_file in rejected_files:
+        try:
+            rejected_file.unlink()
+            logger.debug(f"Deleted rejected {file_type_label} file: {rejected_file.name}")
+        except OSError as e:
+            logger.warning(f"Failed to delete rejected {file_type_label} file {rejected_file}: {e}")
+
+    if rejected_files:
+        rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+        warnings.append(f"Skipped {len(rejected_files)} {file_type_label}(s) with unsupported format: {', '.join(rejected_exts)}")
+
+    # Delete other files (images, html, etc)
+    for other_file in other_files:
+        try:
+            other_file.unlink()
+            logger.debug(f"Deleted non-{file_type_label} file: {other_file.name}")
+        except OSError as e:
+            logger.warning(f"Failed to delete non-{file_type_label} file {other_file}: {e}")
+
+    if other_files:
+        warnings.append(f"Skipped {len(other_files)} non-{file_type_label} file(s)")
+
+    return matched_files, warnings, rejected_files
+
+
+def extract_archive_raw(
+    archive_path: Path,
+    output_dir: Path,
+) -> Tuple[List[Path], List[str]]:
+    """Extract archive without filtering (returns all extracted files)."""
+    suffix = archive_path.suffix.lower().lstrip(".")
+
+    if suffix == "zip":
+        return _extract_zip(archive_path, output_dir)
+    if suffix == "rar":
+        return _extract_rar(archive_path, output_dir)
+
+    raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
+
+
+def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
+    """Extract files from ZipFile or RarFile to output_dir with security checks."""
+    extracted_files = []
+
+    for info in archive.infolist():
+        if info.is_dir():
+            continue
+
+        # Use only filename, strip directory path (security: prevent path traversal)
+        filename = Path(info.filename).name
+        if not filename:
+            continue
+
+        # Security: reject filenames with null bytes or path separators
+        # Check both / and \ since archives may be created on different OSes
+        if "\x00" in filename or "/" in filename or "\\" in filename:
+            logger.warning(f"Skipping suspicious filename in archive: {info.filename!r}")
+            continue
+
+        # Extract to output_dir with flat structure
+        target_path = output_dir / filename
+
+        # Security: verify resolved path stays within output directory (defense-in-depth)
+        try:
+            target_path.resolve().relative_to(output_dir.resolve())
+        except ValueError:
+            logger.warning(f"Path traversal attempt blocked: {info.filename!r}")
+            continue
+
+        with archive.open(info) as src:
+            data = src.read()
+        final_path = atomic_write(target_path, data)
+        extracted_files.append(final_path)
+        logger.debug(f"Extracted: {filename}")
+
+    return extracted_files
+
+
+def _extract_zip(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
+    """Extract files from a ZIP archive."""
+    try:
+        with zipfile.ZipFile(archive_path, "r") as zf:
+            # Check for password protection
+            for info in zf.infolist():
+                if info.flag_bits & 0x1:  # Encrypted flag
+                    raise PasswordProtectedError("ZIP archive is password protected")
+
+            # Test archive integrity
+            bad_file = zf.testzip()
+            if bad_file:
+                raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
+
+            return _extract_files_from_archive(zf, output_dir), []
+
+    except zipfile.BadZipFile as e:
+        raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
+    except PermissionError as e:
+        raise ArchiveExtractionError(f"Permission denied: {e}")
+
+
+def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
+    """Extract files from a RAR archive."""
+    if not RAR_AVAILABLE:
+        raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
+
+    try:
+        with rarfile.RarFile(archive_path, "r") as rf:
+            # Check for password protection
+            if rf.needs_password():
+                raise PasswordProtectedError("RAR archive is password protected")
+
+            # Test archive integrity
+            rf.testrar()
+
+            return _extract_files_from_archive(rf, output_dir), []
+
+    except rarfile.BadRarFile as e:
+        raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
+    except rarfile.RarCannotExec:
+        raise ArchiveExtractionError("unrar binary not found - install unrar package")
+    except PermissionError as e:
+        raise ArchiveExtractionError(f"Permission denied: {e}")
+
+

shelfmark/download/fs.py

@@ -0,0 +1,409 @@
+"""Atomic filesystem operations for concurrent-safe file handling.
+
+These utilities handle file collisions atomically, avoiding TOCTOU race conditions
+when multiple workers may try to write to the same path simultaneously.
+"""
+
+import errno
+import os
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.permissions_debug import log_transfer_permission_context
+
+logger = setup_logger(__name__)
+
+
+
+_VERIFY_IO_WAIT_SECONDS = 3.0
+
+
+def _verify_transfer_size(
+    dest: Path,
+    expected_size: int,
+    action: str,
+) -> None:
+    """Verify file transfer completed successfully.
+
+    Some filesystems (especially remote NAS/CIFS/NFS) can report stale sizes briefly
+    after large writes. Do a second stat after a short delay before declaring failure.
+    """
+    actual_size = dest.stat().st_size
+    if actual_size == expected_size:
+        return
+
+    logger.debug(
+        f"File {action} size mismatch, waiting for filesystem sync: {dest} "
+        f"({actual_size} != {expected_size})"
+    )
+    time.sleep(_VERIFY_IO_WAIT_SECONDS)
+
+    actual_size = dest.stat().st_size
+    if actual_size != expected_size:
+        raise IOError(
+            f"File {action} incomplete, data loss may have occurred. "
+            f"'{dest}' was {actual_size} bytes instead of expected {expected_size}."
+        )
+
+
+def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
+    """Write data to a file with atomic collision detection.
+
+    If the destination already exists, retries with counter suffix (_1, _2, etc.)
+    until a unique path is found.
+
+    Args:
+        dest_path: Desired destination path
+        data: Bytes to write
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually written (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            # O_CREAT | O_EXCL fails atomically if file exists
+            fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+            try:
+                os.write(fd, data)
+            finally:
+                os.close(fd)
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            continue
+
+    raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}")
+
+
+def _is_permission_error(e: Exception) -> bool:
+    """Check if exception is a permission error (including NFS/SMB issues)."""
+    return isinstance(e, PermissionError) or (isinstance(e, OSError) and e.errno == errno.EPERM)
+
+
+def _system_op(op: str, source: Path, dest: Path) -> None:
+    """Execute system command (mv or cp) as final fallback."""
+    logger.warning("Attempting system %s as final fallback: %s -> %s", op, source, dest)
+    subprocess.run(
+        [op, "-f", str(source), str(dest)],
+        check=True,
+        capture_output=True,
+        text=True
+    )
+
+
+def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
+    """Handle NFS/SMB permission errors by falling back to copyfile -> system op."""
+    expected_size = source.stat().st_size
+
+    try:
+        # Fallback 1: copy content only
+        shutil.copyfile(str(source), str(dest))
+        _verify_transfer_size(dest, expected_size, "copy")
+
+        if is_move:
+            source.unlink()
+        return
+
+    except Exception as copy_error:
+        # Clean up failed copy attempt if it exists
+        dest.unlink(missing_ok=True)
+
+        if _is_permission_error(copy_error):
+            log_transfer_permission_context("nfs_fallback_copyfile", source=source, dest=dest, error=copy_error)
+        logger.error("Fallback copyfile failed (%s -> %s): %s", source, dest, copy_error)
+
+        # Fallback 2: system command
+        op = "mv" if is_move else "cp"
+        try:
+            _system_op(op, source, dest)
+            # Best-effort verify after external command.
+            if dest.exists():
+                _verify_transfer_size(dest, expected_size, op)
+            if is_move:
+                source.unlink(missing_ok=True)
+        except subprocess.CalledProcessError as sys_error:
+            log_transfer_permission_context("nfs_fallback_system", source=source, dest=dest, error=sys_error)
+            logger.error("System %s failed (%s -> %s): %s", op, source, dest, sys_error.stderr)
+            dest.unlink(missing_ok=True)
+            raise
+
+
+def _claim_destination(path: Path) -> bool:
+    """Atomically claim a destination path by creating a placeholder file.
+
+    Returns True if the placeholder was created. Caller must replace or unlink it.
+    """
+    try:
+        fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+    except FileExistsError:
+        return False
+    else:
+        os.close(fd)
+        return True
+
+
+def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Move a file with collision detection.
+
+    Uses os.rename() for same-filesystem moves (atomic, triggers inotify events),
+    falls back to exclusive create + shutil.move for cross-filesystem moves.
+
+    Note: We use os.rename() instead of hardlink+unlink because os.rename()
+    triggers proper inotify IN_MOVED_TO events that file watchers (like Calibre's
+    auto-add) rely on to detect new files.
+
+    Args:
+        source_path: Source file to move
+        dest_path: Desired destination path
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually moved (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+
+        # Check for existing file (os.rename would overwrite on Unix)
+        claimed = False
+        if try_path.exists():
+            # Some filesystems can report false positives for exists() with
+            # special characters. Probe with O_EXCL to confirm.
+            claimed = _claim_destination(try_path)
+            if not claimed:
+                continue
+
+        try:
+            # os.rename is atomic on same filesystem and triggers inotify events
+            if claimed:
+                os.replace(str(source_path), str(try_path))
+            else:
+                os.rename(str(source_path), str(try_path))
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            # Race condition: file created between exists() check and rename()
+            if claimed:
+                try_path.unlink(missing_ok=True)
+            continue
+        except OSError as e:
+            # Cross-filesystem - fall back to exclusive create + verified copy + delete.
+            if e.errno != errno.EXDEV:
+                if claimed:
+                    try_path.unlink(missing_ok=True)
+                raise
+
+            expected_size = source_path.stat().st_size
+
+            try:
+                if not claimed:
+                    # Claim destination path atomically.
+                    fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+                    os.close(fd)
+
+                # Copy to a temp file first, then replace to avoid partial files.
+                temp_path = try_path.parent / f".{try_path.name}.tmp"
+                try:
+                    try:
+                        shutil.copy2(str(source_path), str(temp_path))
+                    except (PermissionError, OSError) as copy_error:
+                        if _is_permission_error(copy_error):
+                            logger.debug(
+                                "Permission error during move-copy, falling back to copyfile (%s -> %s): %s",
+                                source_path,
+                                temp_path,
+                                copy_error,
+                            )
+                            _perform_nfs_fallback(source_path, temp_path, is_move=False)
+                        else:
+                            raise
+
+                    temp_path.replace(try_path)
+                    _verify_transfer_size(try_path, expected_size, "move")
+                    source_path.unlink()
+
+                    if attempt > 0:
+                        logger.info(f"File collision resolved: {try_path.name}")
+                    return try_path
+
+                except Exception:
+                    try_path.unlink(missing_ok=True)
+                    temp_path.unlink(missing_ok=True)
+                    raise
+
+            except FileExistsError:
+                continue
+            except (PermissionError, OSError) as e:
+                if _is_permission_error(e):
+                    log_transfer_permission_context(
+                        "atomic_move",
+                        source=source_path,
+                        dest=try_path,
+                        error=e,
+                    )
+                    logger.debug(
+                        "Permission error during move, falling back to copyfile (%s -> %s): %s",
+                        source_path,
+                        try_path,
+                        e,
+                    )
+                    try:
+                        _perform_nfs_fallback(source_path, try_path, is_move=True)
+                        if attempt > 0:
+                            logger.info(f"File collision resolved (fallback): {try_path.name}")
+                        return try_path
+                    except Exception as fallback_error:
+                        logger.error(
+                            "NFS fallback also failed (%s -> %s): %s",
+                            source_path,
+                            try_path,
+                            fallback_error,
+                        )
+                        raise e from fallback_error
+                raise
+
+    raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
+
+
+def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Create a hardlink with atomic collision detection.
+
+    Args:
+        source_path: Source file to link from
+        dest_path: Desired destination path for the link
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where link was actually created (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            os.link(str(source_path), str(try_path))
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            continue
+        except OSError as e:
+            if _is_permission_error(e) or e.errno in (errno.EXDEV, errno.EMLINK):
+                if _is_permission_error(e):
+                    log_transfer_permission_context(
+                        "atomic_hardlink",
+                        source=source_path,
+                        dest=try_path,
+                        error=e,
+                    )
+                logger.debug(
+                    "Hardlink failed (%s), falling back to copy: %s -> %s",
+                    e,
+                    source_path,
+                    dest_path,
+                )
+                return atomic_copy(source_path, dest_path, max_attempts=max_attempts)
+            raise
+
+    raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
+
+
+def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Copy a file with atomic collision detection.
+
+    Uses exclusive create to claim destination, then copies via temp file
+    to avoid partial files on failure.
+
+    Args:
+        source_path: Source file to copy
+        dest_path: Desired destination path
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually copied (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            # Atomically claim the destination by creating an exclusive file
+            fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+            os.close(fd)
+            
+            # Copy to temp file first, then replace to avoid partial files
+            temp_path = try_path.parent / f".{try_path.name}.tmp"
+            try:
+                try:
+                    shutil.copy2(str(source_path), str(temp_path))
+                except (PermissionError, OSError) as e:
+                    # Handle NFS permission errors immediately here
+                    if _is_permission_error(e):
+                        log_transfer_permission_context(
+                            "atomic_copy",
+                            source=source_path,
+                            dest=temp_path,
+                            error=e,
+                        )
+                        logger.debug(
+                            "Permission error during copy, falling back to copyfile (%s -> %s): %s",
+                            source_path,
+                            temp_path,
+                            e,
+                        )
+                        try:
+                            _perform_nfs_fallback(source_path, temp_path, is_move=False)
+                        except Exception as fallback_error:
+                            logger.error(
+                                "NFS fallback also failed (%s -> %s): %s",
+                                source_path,
+                                temp_path,
+                                fallback_error,
+                            )
+                            raise e from fallback_error
+                    else:
+                        raise
+                
+                temp_path.replace(try_path)
+                _verify_transfer_size(try_path, source_path.stat().st_size, "copy")
+                if attempt > 0:
+                    logger.info(f"File collision resolved: {try_path.name}")
+                return try_path
+            except Exception:
+                try_path.unlink(missing_ok=True)
+                temp_path.unlink(missing_ok=True)
+                raise
+        except FileExistsError:
+            continue
+
+    raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")

shelfmark/download/http.py

@@ -0,0 +1,465 @@
+"""HTTP download with retry, resume, and Cloudflare bypass support."""
+
+import random
+import time
+from io import BytesIO
+from threading import Event
+from typing import Callable, Optional
+from urllib.parse import urlparse
+
+import requests
+from tqdm import tqdm
+
+from shelfmark.download import network
+from shelfmark.download.network import get_proxies
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Bypasser modules are imported lazily to support dynamic selection based on config
+_internal_bypasser = None
+_external_bypasser = None
+
+
+def _get_internal_bypasser():
+    """Lazy import of internal bypasser module."""
+    global _internal_bypasser
+    if _internal_bypasser is None:
+        try:
+            from shelfmark.bypass import internal_bypasser
+            _internal_bypasser = internal_bypasser
+        except ImportError as e:
+            raise RuntimeError(
+                f"Failed to import internal bypasser: {e}. "
+                "Check that all dependencies are installed. "
+                "You may need to disable CF bypass or use the external bypasser."
+            ) from e
+    return _internal_bypasser
+
+
+def _get_external_bypasser():
+    """Lazy import of external bypasser module."""
+    global _external_bypasser
+    if _external_bypasser is None:
+        try:
+            from shelfmark.bypass import external_bypasser
+            _external_bypasser = external_bypasser
+        except ImportError as e:
+            raise RuntimeError(
+                f"Failed to import external bypasser: {e}. "
+                "Check that the external bypasser is properly configured."
+            ) from e
+    return _external_bypasser
+
+
+def _is_using_external_bypasser() -> bool:
+    """Check if external bypasser is configured (reads from config, not just env)."""
+    return app_config.get("USING_EXTERNAL_BYPASSER", False)
+
+
+def _is_cf_bypass_enabled() -> bool:
+    """Check if Cloudflare bypass is enabled."""
+    return app_config.get("USE_CF_BYPASS", True)
+
+
+def get_bypassed_page(url, selector=None, cancel_flag=None):
+    """Wrapper that delegates to the appropriate bypasser based on config."""
+    if _is_using_external_bypasser():
+        return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag)
+    return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
+
+
+def get_cf_cookies_for_domain(domain):
+    """Get CF cookies - only available with internal bypasser."""
+    if _is_using_external_bypasser():
+        logger.debug(f"External bypasser in use, CF cookies not available for {domain}")
+        return {}
+    return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
+
+
+def get_cf_user_agent_for_domain(domain):
+    """Get CF user agent - only available with internal bypasser."""
+    if _is_using_external_bypasser():
+        logger.debug(f"External bypasser in use, CF user agent not available for {domain}")
+        return None
+    return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
+
+
+def _apply_cf_bypass(url: str, headers: dict) -> dict:
+    """Apply CF bypass cookies and user agent if available.
+
+    Modifies headers in-place with the stored user agent (if available).
+    Returns cookies dict to use with the request.
+    """
+    if not _is_cf_bypass_enabled():
+        return {}
+
+    parsed = urlparse(url)
+    hostname = parsed.hostname or ""
+    cookies = get_cf_cookies_for_domain(hostname)
+    stored_ua = get_cf_user_agent_for_domain(hostname)
+    if stored_ua:
+        headers['User-Agent'] = stored_ua
+    return cookies
+
+
+# Network settings
+REQUEST_TIMEOUT = (5, 10)  # (connect, read)
+MAX_DOWNLOAD_RETRIES = 2
+MAX_RESUME_ATTEMPTS = 3
+
+RETRYABLE_CODES = (429, 500, 502, 503, 504)
+CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout,
+                     requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError)
+DOWNLOAD_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+}
+
+
+def parse_size_string(size: str) -> Optional[float]:
+    """Parse a human-readable size string (e.g., '10.5 MB') into bytes."""
+    if not size:
+        return None
+    try:
+        normalized = size.strip().replace(" ", "").replace(",", ".").upper()
+        multipliers = {"GB": 1024**3, "MB": 1024**2, "KB": 1024}
+        for suffix, mult in multipliers.items():
+            if normalized.endswith(suffix):
+                return float(normalized[:-2]) * mult
+        return float(normalized)
+    except (ValueError, IndexError):
+        return None
+
+def _backoff_delay(attempt: int, base: float = 0.25, cap: float = 3.0) -> float:
+    """Exponential backoff with jitter."""
+    return min(cap, base * (2 ** (attempt - 1))) + random.random() * base
+
+
+def _get_status_code(e: Exception) -> Optional[int]:
+    """Extract HTTP status code from an exception, or None if not applicable."""
+    if isinstance(e, requests.exceptions.HTTPError) and e.response is not None:
+        return e.response.status_code
+    return None
+
+def _is_retryable_error(e: Exception) -> bool:
+    """Check if error is retryable (connection error or retryable HTTP status)."""
+    if isinstance(e, CONNECTION_ERRORS):
+        return True
+    status = _get_status_code(e)
+    return status is not None and status in RETRYABLE_CODES
+
+
+def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]:
+    """Try mirror/DNS rotation. Returns new URL or None."""
+    if current_url.startswith(network.get_aa_base_url()):
+        new_base, action = selector.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            new_url = selector.rewrite(original_url)
+            logger.info(f"[{action}] switching to: {new_url}")
+            return new_url
+    elif network.should_rotate_dns_for_url(current_url) and network.rotate_dns_provider():
+        logger.info(f"[dns-rotate] retrying: {original_url}")
+        return original_url
+    return None
+
+
+def html_get_page(
+    url: str,
+    retry: Optional[int] = None,
+    use_bypasser: bool = False,
+    selector: Optional[network.AAMirrorSelector] = None,
+    cancel_flag: Optional[Event] = None,
+    status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
+    allow_bypasser_fallback: bool = True,
+) -> str:
+    """Fetch HTML content from a URL with retry mechanism.
+
+    Args:
+        allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
+            instead of switching to the bypasser. Use for search operations.
+    """
+    retry = retry if retry is not None else app_config.MAX_RETRY
+    selector = selector or network.AAMirrorSelector()
+    original_url = url
+    current_url = selector.rewrite(original_url)
+    use_bypasser_now = use_bypasser
+
+    for attempt in range(1, retry + 1):
+        # Check for cancellation before each attempt
+        if cancel_flag and cancel_flag.is_set():
+            logger.info(f"html_get_page cancelled before attempt {attempt}")
+            return ""
+
+        try:
+            if use_bypasser_now and _is_cf_bypass_enabled():
+                logger.debug(f"GET (bypasser): {current_url}")
+                if status_callback:
+                    status_callback("resolving", "Bypassing protection")
+                try:
+                    result = get_bypassed_page(current_url, selector, cancel_flag)
+                    return result or ""
+                except Exception as e:
+                    logger.warning(f"Bypasser error: {type(e).__name__}: {e}")
+                    return ""
+
+            logger.debug(f"GET: {current_url}")
+            # Try with CF cookies/UA if available (from previous bypass)
+            headers = {}
+            cookies = _apply_cf_bypass(current_url, headers)
+            response = requests.get(current_url, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
+            response.raise_for_status()
+            time.sleep(1)
+            return response.text
+
+        except Exception as e:
+            status = _get_status_code(e)
+
+            # 403 = Cloudflare/DDoS-Guard protection
+            if status == 403:
+                # If bypasser fallback is disabled, try mirrors instead
+                if not allow_bypasser_fallback:
+                    new_url = _try_rotation(original_url, current_url, selector)
+                    if new_url:
+                        current_url = new_url
+                        continue
+                    logger.warning(f"403 error, mirrors exhausted: {current_url}")
+                    return ""
+
+                if _is_cf_bypass_enabled() and not use_bypasser_now:
+                    # Before switching to bypasser, check if cookies have become available
+                    # (another concurrent download may have completed bypass and extracted cookies)
+                    parsed = urlparse(current_url)
+                    fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "")
+                    if fresh_cookies and not cookies:
+                        # Cookies are now available - retry with cookies before using bypasser
+                        logger.debug(f"403 but cookies now available - retrying with cookies: {current_url}")
+                        continue
+                    logger.info(f"403 detected; switching to bypasser: {current_url}")
+                    if status_callback:
+                        status_callback("resolving", "Bypassing protection...")
+                    use_bypasser_now = True
+                    continue
+                logger.warning(f"403 error, giving up: {current_url}")
+                return ""
+
+            # 404 = Not found
+            if status == 404:
+                logger.warning(f"404 error: {current_url}")
+                return ""
+
+            # Try mirror/DNS rotation on retryable errors
+            if _is_retryable_error(e):
+                new_url = _try_rotation(original_url, current_url, selector)
+                if new_url:
+                    current_url = new_url
+                    continue
+
+            # Retry with backoff
+            if attempt < retry:
+                logger.warning(f"Retry {attempt}/{retry} for {current_url}: {type(e).__name__}: {e}")
+                time.sleep(_backoff_delay(attempt))
+            else:
+                logger.error(f"Giving up after {retry} attempts: {current_url}")
+
+    return ""
+
+
+def download_url(
+    link: str,
+    size: str = "",
+    progress_callback: Optional[Callable[[float], None]] = None,
+    cancel_flag: Optional[Event] = None,
+    _selector: Optional[network.AAMirrorSelector] = None,
+    status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
+    referer: Optional[str] = None,
+) -> Optional[BytesIO]:
+    """Download content from URL with automatic retry and resume support."""
+    selector = _selector or network.AAMirrorSelector()
+    current_url = selector.rewrite(link)
+
+    # Build headers with optional referer
+    headers = DOWNLOAD_HEADERS.copy()
+    if referer:
+        headers['Referer'] = referer
+    total_size = parse_size_string(size) or 0
+
+    attempt = 0
+    zlib_cookie_refresh_attempted = False
+
+    while attempt < MAX_DOWNLOAD_RETRIES:
+        if cancel_flag and cancel_flag.is_set():
+            return None
+
+        buffer = BytesIO()
+        bytes_downloaded = 0
+
+        try:
+            if attempt > 0 and status_callback:
+                status_callback("resolving", f"Connecting (Attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
+
+            logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
+            # Try with CF cookies/UA if available
+            cookies = _apply_cf_bypass(current_url, headers)
+            response = requests.get(current_url, stream=True, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
+            response.raise_for_status()
+
+            if status_callback:
+                status_callback("downloading", "")
+
+            total_size = total_size or float(response.headers.get('content-length', 0))
+            pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
+
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    buffer.write(chunk)
+                    bytes_downloaded += len(chunk)
+                    pbar.update(len(chunk))
+                    if progress_callback and total_size > 0:
+                        progress_callback(bytes_downloaded * 100.0 / total_size)
+                    if cancel_flag and cancel_flag.is_set():
+                        pbar.close()
+                        return None
+            pbar.close()
+
+            # Validate - check we didn't get HTML instead of file
+            if total_size > 0 and bytes_downloaded < total_size * 0.9:
+                if response.headers.get('content-type', '').startswith('text/html'):
+                    logger.warning(f"Received HTML instead of file: {current_url}")
+                    return None
+
+            logger.debug(f"Download completed: {bytes_downloaded} bytes")
+            return buffer
+
+        except requests.exceptions.RequestException as e:
+            status = _get_status_code(e)
+            retryable = _is_retryable_error(e)
+
+            # Z-Library 403 - try refreshing cookies via bypasser once before giving up
+            if status == 403 and _is_cf_bypass_enabled() and not zlib_cookie_refresh_attempted:
+                parsed = urlparse(current_url)
+                if parsed.hostname and 'z-lib' in parsed.hostname and referer:
+                    zlib_cookie_refresh_attempted = True
+                    logger.info(f"Z-Library 403 - refreshing cookies via referer: {referer}")
+                    try:
+                        get_bypassed_page(referer, selector, cancel_flag)
+                        time.sleep(0.5)
+                        # Retry with fresh cookies (don't increment attempt)
+                        continue
+                    except Exception as cookie_err:
+                        logger.warning(f"Z-Library cookie refresh failed: {cookie_err}")
+
+            # Non-retryable errors
+            if status in (403, 404):
+                logger.warning(f"Download failed ({status}): {current_url}")
+                return None
+
+            # Rate limited - skip to next source immediately
+            # (waiting doesn't help with concurrent downloads hitting the same server)
+            if status == 429:
+                logger.info(f"Rate limited (429) - trying next source")
+                if status_callback:
+                    status_callback("resolving", "Server busy, trying next")
+                return None
+
+            # Timeout - don't retry, server likely overloaded
+            if isinstance(e, requests.exceptions.Timeout):
+                logger.warning(f"Timeout: {current_url} - skipping to next source")
+                if status_callback:
+                    status_callback("resolving", "Server timed out, trying next")
+                return None
+
+            # Try to resume if we got some data
+            if bytes_downloaded > 0 and retryable:
+                resumed = _try_resume(current_url, buffer, bytes_downloaded, total_size, progress_callback, cancel_flag, headers)
+                if resumed:
+                    return resumed
+
+            # Try mirror/DNS rotation if nothing downloaded yet
+            if bytes_downloaded == 0 and retryable:
+                new_url = _try_rotation(link, current_url, selector)
+                if new_url:
+                    current_url = new_url
+                    attempt += 1
+                    continue
+
+            logger.warning(f"Download error: {type(e).__name__}: {e}")
+            if attempt < MAX_DOWNLOAD_RETRIES - 1:
+                time.sleep(_backoff_delay(attempt + 1))
+            attempt += 1
+
+    logger.error(f"Download failed after {MAX_DOWNLOAD_RETRIES} attempts: {link}")
+    return None
+
+
+def _try_resume(
+    url: str,
+    buffer: BytesIO,
+    start_byte: int,
+    total_size: float,
+    progress_callback: Optional[Callable[[float], None]],
+    cancel_flag: Optional[Event],
+    base_headers: Optional[dict] = None,
+) -> Optional[BytesIO]:
+    """Try to resume an interrupted download."""
+    for attempt in range(MAX_RESUME_ATTEMPTS):
+        logger.info(f"Resuming from {start_byte} bytes (attempt {attempt + 1}/{MAX_RESUME_ATTEMPTS})")
+        time.sleep(_backoff_delay(attempt + 1, base=0.5, cap=5.0))
+
+        try:
+            # Try with CF cookies/UA if available
+            resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'}
+            cookies = _apply_cf_bypass(url, resume_headers)
+            response = requests.get(
+                url, stream=True, proxies=get_proxies(url), timeout=REQUEST_TIMEOUT,
+                headers=resume_headers, cookies=cookies
+            )
+            
+            # Check resume support
+            if response.status_code == 200:  # Server doesn't support resume
+                logger.info("Server doesn't support resume")
+                return None
+            if response.status_code == 416:  # Range not satisfiable
+                logger.warning("Range not satisfiable")
+                return None
+            if response.status_code != 206:
+                response.raise_for_status()
+            
+            pbar = tqdm(total=total_size, initial=start_byte, unit='B', unit_scale=True, desc='Resuming')
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    buffer.write(chunk)
+                    start_byte += len(chunk)
+                    pbar.update(len(chunk))
+                    if progress_callback and total_size > 0:
+                        progress_callback(start_byte * 100.0 / total_size)
+                    if cancel_flag and cancel_flag.is_set():
+                        pbar.close()
+                        return None
+            pbar.close()
+            
+            logger.info(f"Resume completed: {start_byte} bytes")
+            return buffer
+            
+        except requests.exceptions.RequestException as e:
+            logger.debug(f"Resume attempt {attempt + 1} failed: {e}")
+    
+    logger.warning(f"Resume failed after {MAX_RESUME_ATTEMPTS} attempts")
+    return None
+
+
+def get_absolute_url(base_url: str, url: str) -> str:
+    """Convert a relative URL to absolute using the base URL."""
+    url = url.strip()
+    if not url or url == "#" or url.startswith("http"):
+        return url if url.startswith("http") else ""
+    parsed = urlparse(url)
+    base = urlparse(base_url)
+    if not parsed.netloc or not parsed.scheme:
+        parsed = parsed._replace(netloc=base.netloc, scheme=base.scheme)
+    return parsed.geturl()

shelfmark/download/orchestrator.py

@@ -0,0 +1,575 @@
+"""Download queue orchestration and worker management.
+
+Two-stage architecture: handlers stage to TMP_DIR, orchestrator moves to INGEST_DIR
+with archive extraction and custom script support.
+"""
+
+import os
+import random
+import threading
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from threading import Event, Lock
+from typing import Any, Dict, List, Optional, Tuple
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters, SearchMode
+from shelfmark.core.queue import book_queue
+from shelfmark.core.utils import transform_cover_url
+from shelfmark.download.postprocess.pipeline import is_torrent_source, safe_cleanup_path
+from shelfmark.download.postprocess.router import post_process_download
+from shelfmark.release_sources import direct_download, get_handler, get_source_display_name
+from shelfmark.release_sources.direct_download import SearchUnavailable
+
+logger = setup_logger(__name__)
+
+
+# =============================================================================
+# Task Download and Processing
+# =============================================================================
+#
+# Post-download processing (staging, extraction, transfers, cleanup) lives in
+# `shelfmark.download.postprocess`.
+
+
+# WebSocket manager (initialized by app.py)
+# Track whether WebSocket is available for status reporting
+WEBSOCKET_AVAILABLE = True
+try:
+    from shelfmark.api.websocket import ws_manager
+except ImportError:
+    logger.error("WebSocket unavailable - real-time updates disabled")
+    ws_manager = None
+    WEBSOCKET_AVAILABLE = False
+
+# Progress update throttling - track last broadcast time per book
+_progress_last_broadcast: Dict[str, float] = {}
+_progress_lock = Lock()
+
+# Stall detection - track last activity time per download
+_last_activity: Dict[str, float] = {}
+STALL_TIMEOUT = 300  # 5 minutes without progress/status update = stalled
+
+def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
+    """Search for books matching the query."""
+    try:
+        books = direct_download.search_books(query, filters)
+        return [_book_info_to_dict(book) for book in books]
+    except SearchUnavailable:
+        raise
+    except Exception as e:
+        logger.error_trace(f"Error searching books: {e}")
+        raise
+
+def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
+    """Get detailed information for a specific book."""
+    try:
+        book = direct_download.get_book_info(book_id)
+        return _book_info_to_dict(book)
+    except Exception as e:
+        logger.error_trace(f"Error getting book info: {e}")
+        raise
+
+def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> Tuple[bool, Optional[str]]:
+    """Add a book to the download queue. Returns (success, error_message)."""
+    try:
+        book_info = direct_download.get_book_info(book_id, fetch_download_count=False)
+        if not book_info:
+            error_msg = f"Could not fetch book info for {book_id}"
+            logger.warning(error_msg)
+            return False, error_msg
+
+        # Create a source-agnostic download task
+        task = DownloadTask(
+            task_id=book_id,
+            source=source,
+            title=book_info.title,
+            author=book_info.author,
+            format=book_info.format,
+            size=book_info.size,
+            preview=book_info.preview,
+            content_type=book_info.content,
+            search_mode=SearchMode.DIRECT,
+            priority=priority,
+        )
+
+        if not book_queue.add(task):
+            logger.info(f"Book already in queue: {book_info.title}")
+            return False, "Book is already in the download queue"
+
+        logger.info(f"Book queued with priority {priority}: {book_info.title}")
+
+        # Broadcast status update via WebSocket
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+        return True, None
+    except SearchUnavailable as e:
+        error_msg = f"Search service unavailable: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except Exception as e:
+        error_msg = f"Error queueing book: {e}"
+        logger.error_trace(error_msg)
+        return False, error_msg
+
+
+def queue_release(release_data: dict, priority: int = 0) -> Tuple[bool, Optional[str]]:
+    """Add a release to the download queue. Returns (success, error_message)."""
+    try:
+        source = release_data.get('source', 'direct_download')
+        extra = release_data.get('extra', {})
+
+        # Get author, year, preview, and content_type from top-level (preferred) or extra (fallback)
+        author = release_data.get('author') or extra.get('author')
+        year = release_data.get('year') or extra.get('year')
+        preview = release_data.get('preview') or extra.get('preview')
+        content_type = release_data.get('content_type') or extra.get('content_type')
+
+        # Get series info for library naming templates
+        series_name = release_data.get('series_name') or extra.get('series_name')
+        series_position = release_data.get('series_position') or extra.get('series_position')
+        subtitle = release_data.get('subtitle') or extra.get('subtitle')
+
+        # Create a source-agnostic download task from release data
+        task = DownloadTask(
+            task_id=release_data['source_id'],
+            source=source,
+            title=release_data.get('title', 'Unknown'),
+            author=author,
+            year=year,
+            format=release_data.get('format'),
+            size=release_data.get('size'),
+            preview=preview,
+            content_type=content_type,
+            series_name=series_name,
+            series_position=series_position,
+            subtitle=subtitle,
+            search_mode=SearchMode.UNIVERSAL,
+            priority=priority,
+        )
+
+        if not book_queue.add(task):
+            logger.info(f"Release already in queue: {task.title}")
+            return False, "Release is already in the download queue"
+
+        logger.info(f"Release queued with priority {priority}: {task.title}")
+
+        # Broadcast status update via WebSocket
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+        return True, None
+
+    except ValueError as e:
+        # Handler not found for this source
+        error_msg = f"Unknown release source: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except KeyError as e:
+        error_msg = f"Missing required field in release data: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except Exception as e:
+        error_msg = f"Error queueing release: {e}"
+        logger.error_trace(error_msg)
+        return False, error_msg
+
+def queue_status() -> Dict[str, Dict[str, Any]]:
+    """Get current status of the download queue."""
+    status = book_queue.get_status()
+    for _, tasks in status.items():
+        for _, task in tasks.items():
+            if task.download_path and not os.path.exists(task.download_path):
+                task.download_path = None
+
+    # Convert Enum keys to strings and DownloadTask objects to dicts for JSON serialization
+    return {
+        status_type.value: {
+            task_id: _task_to_dict(task)
+            for task_id, task in tasks.items()
+        }
+        for status_type, tasks in status.items()
+    }
+
+def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]]:
+    """Get downloaded file data for a specific task."""
+    task = None
+    try:
+        task = book_queue.get_task(task_id)
+        if not task:
+            return None, None
+
+        path = task.download_path
+        if not path:
+            return None, task
+
+        with open(path, "rb") as f:
+            return f.read(), task
+    except Exception as e:
+        logger.error_trace(f"Error getting book data: {e}")
+        if task:
+            task.download_path = None
+        return None, task
+
+def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
+    """Convert BookInfo to dict, transforming cover URLs for caching."""
+    result = {
+        key: value for key, value in book.__dict__.items()
+        if value is not None
+    }
+
+    # Transform external preview URLs to local proxy URLs
+    if result.get('preview'):
+        result['preview'] = transform_cover_url(result['preview'], book.id)
+
+    return result
+
+
+def _task_to_dict(task: DownloadTask) -> Dict[str, Any]:
+    """Convert DownloadTask to dict for frontend, transforming cover URLs."""
+    # Transform external preview URLs to local proxy URLs
+    preview = transform_cover_url(task.preview, task.task_id)
+
+    return {
+        'id': task.task_id,
+        'title': task.title,
+        'author': task.author,
+        'format': task.format,
+        'size': task.size,
+        'preview': preview,
+        'content_type': task.content_type,
+        'source': task.source,
+        'source_display_name': get_source_display_name(task.source),
+        'priority': task.priority,
+        'added_time': task.added_time,
+        'progress': task.progress,
+        'status': task.status,
+        'status_message': task.status_message,
+        'download_path': task.download_path,
+    }
+
+
+def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
+    """Download a task via appropriate handler, then post-process to ingest."""
+    try:
+        # Check for cancellation before starting
+        if cancel_flag.is_set():
+            logger.info("Task %s: cancelled before starting", task_id)
+            return None
+
+        task = book_queue.get_task(task_id)
+        if not task:
+            logger.error("Task not found in queue: %s", task_id)
+            return None
+
+        title_label = task.title or "Unknown title"
+        logger.info(
+            "Task %s: starting download (%s) - %s",
+            task_id,
+            get_source_display_name(task.source),
+            title_label,
+        )
+
+        def progress_callback(progress: float) -> None:
+            update_download_progress(task_id, progress)
+
+        def status_callback(status: str, message: Optional[str] = None) -> None:
+            update_download_status(task_id, status, message)
+
+        # Get the download handler based on the task's source
+        handler = get_handler(task.source)
+        temp_path = handler.download(
+            task,
+            cancel_flag,
+            progress_callback,
+            status_callback
+        )
+
+        # Handler returns temp path - orchestrator handles post-processing
+        if not temp_path:
+            return None
+
+        temp_file = Path(temp_path)
+        if not temp_file.exists():
+            logger.error(f"Handler returned non-existent path: {temp_path}")
+            return None
+
+        # Check cancellation before post-processing
+        if cancel_flag.is_set():
+            logger.info("Task %s: cancelled before post-processing", task_id)
+            if not is_torrent_source(temp_file, task):
+                safe_cleanup_path(temp_file, task)
+            return None
+
+        logger.info("Task %s: download finished; starting post-processing", task_id)
+        logger.debug("Task %s: post-processing input path: %s", task_id, temp_file)
+
+        # Post-processing: output routing + file processing pipeline
+        result = post_process_download(temp_file, task, cancel_flag, status_callback)
+
+        if cancel_flag.is_set():
+            logger.info("Task %s: post-processing cancelled", task_id)
+        elif result:
+            logger.info("Task %s: post-processing complete", task_id)
+            logger.debug("Task %s: post-processing result: %s", task_id, result)
+        else:
+            logger.warning("Task %s: post-processing failed", task_id)
+
+        try:
+            handler.post_process_cleanup(task, success=bool(result))
+        except Exception as e:
+            logger.warning("Post-processing cleanup hook failed for %s: %s", task_id, e)
+
+        return result
+
+    except Exception as e:
+        if cancel_flag.is_set():
+            logger.info("Task %s: cancelled during error handling", task_id)
+        else:
+            logger.error_trace("Task %s: error downloading: %s", task_id, e)
+            # Update task status so user sees the failure
+            task = book_queue.get_task(task_id)
+            if task:
+                book_queue.update_status(task_id, QueueStatus.ERROR)
+                # Check for known misconfiguration from earlier versions
+                if isinstance(e, PermissionError) and "/cwa-book-ingest" in str(e):
+                    book_queue.update_status_message(
+                        task_id,
+                        "Destination misconfigured. Go to Settings → Downloads to update."
+                    )
+                else:
+                    if isinstance(e, PermissionError):
+                        book_queue.update_status_message(task_id, f"Permission denied: {e}")
+                    else:
+                        book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}")
+        return None
+
+
+
+def update_download_progress(book_id: str, progress: float) -> None:
+    """Update download progress with throttled WebSocket broadcasts."""
+    book_queue.update_progress(book_id, progress)
+
+    # Track activity for stall detection
+    with _progress_lock:
+        _last_activity[book_id] = time.time()
+    
+    # Broadcast progress via WebSocket with throttling
+    if ws_manager:
+        current_time = time.time()
+        should_broadcast = False
+        
+        with _progress_lock:
+            last_broadcast = _progress_last_broadcast.get(book_id, 0)
+            last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0)
+            time_elapsed = current_time - last_broadcast
+            
+            # Always broadcast at start (0%) or completion (>=99%)
+            if progress <= 1 or progress >= 99:
+                should_broadcast = True
+            # Broadcast if enough time has passed (convert interval from seconds)
+            elif time_elapsed >= config.DOWNLOAD_PROGRESS_UPDATE_INTERVAL:
+                should_broadcast = True
+            # Broadcast on significant progress jumps (>10%)
+            elif progress - last_progress >= 10:
+                should_broadcast = True
+            
+            if should_broadcast:
+                _progress_last_broadcast[book_id] = current_time
+                _progress_last_broadcast[f"{book_id}_progress"] = progress
+        
+        if should_broadcast:
+            ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
+
+def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
+    """Update download status with optional message for UI display."""
+    # Map string status to QueueStatus enum
+    status_map = {
+        'queued': QueueStatus.QUEUED,
+        'resolving': QueueStatus.RESOLVING,
+        'downloading': QueueStatus.DOWNLOADING,
+        'complete': QueueStatus.COMPLETE,
+        'available': QueueStatus.AVAILABLE,
+        'error': QueueStatus.ERROR,
+        'done': QueueStatus.DONE,
+        'cancelled': QueueStatus.CANCELLED,
+    }
+    
+    queue_status_enum = status_map.get(status.lower())
+    if queue_status_enum:
+        book_queue.update_status(book_id, queue_status_enum)
+
+        # Track activity for stall detection
+        with _progress_lock:
+            _last_activity[book_id] = time.time()
+
+        # Update status message if provided (empty string clears the message)
+        if message is not None:
+            book_queue.update_status_message(book_id, message)
+
+        # Broadcast status update via WebSocket
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+def cancel_download(book_id: str) -> bool:
+    """Cancel a download."""
+    result = book_queue.cancel_download(book_id)
+    
+    # Broadcast status update via WebSocket
+    if result and ws_manager and ws_manager.is_enabled():
+        ws_manager.broadcast_status_update(queue_status())
+    
+    return result
+
+def set_book_priority(book_id: str, priority: int) -> bool:
+    """Set priority for a queued book (lower = higher priority)."""
+    return book_queue.set_priority(book_id, priority)
+
+def reorder_queue(book_priorities: Dict[str, int]) -> bool:
+    """Bulk reorder queue by mapping book_id to new priority."""
+    return book_queue.reorder_queue(book_priorities)
+
+def get_queue_order() -> List[Dict[str, Any]]:
+    """Get current queue order for display."""
+    return book_queue.get_queue_order()
+
+def get_active_downloads() -> List[str]:
+    """Get list of currently active downloads."""
+    return book_queue.get_active_downloads()
+
+def clear_completed() -> int:
+    """Clear all completed downloads from tracking."""
+    return book_queue.clear_completed()
+
+def _cleanup_progress_tracking(task_id: str) -> None:
+    """Clean up progress tracking data for a completed/cancelled download."""
+    with _progress_lock:
+        _progress_last_broadcast.pop(task_id, None)
+        _progress_last_broadcast.pop(f"{task_id}_progress", None)
+        _last_activity.pop(task_id, None)
+
+
+def _process_single_download(task_id: str, cancel_flag: Event) -> None:
+    """Process a single download job."""
+    try:
+        # Status will be updated through callbacks during download process
+        # (resolving -> downloading -> complete)
+        download_path = _download_task(task_id, cancel_flag)
+
+        # Clean up progress tracking
+        _cleanup_progress_tracking(task_id)
+
+        if cancel_flag.is_set():
+            book_queue.update_status(task_id, QueueStatus.CANCELLED)
+            # Broadcast cancellation
+            if ws_manager:
+                ws_manager.broadcast_status_update(queue_status())
+            return
+
+        if download_path:
+            book_queue.update_download_path(task_id, download_path)
+            # Only update status if not already set (e.g., by archive extraction callback)
+            task = book_queue.get_task(task_id)
+            if not task or task.status != QueueStatus.COMPLETE:
+                book_queue.update_status(task_id, QueueStatus.COMPLETE)
+        else:
+            book_queue.update_status(task_id, QueueStatus.ERROR)
+
+        # Broadcast final status (completed or error)
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+    except Exception as e:
+        # Clean up progress tracking even on error
+        _cleanup_progress_tracking(task_id)
+
+        if not cancel_flag.is_set():
+            logger.error_trace(f"Error in download processing: {e}")
+            book_queue.update_status(task_id, QueueStatus.ERROR)
+            # Set error message if not already set by handler
+            task = book_queue.get_task(task_id)
+            if task and not task.status_message:
+                book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}: {str(e)}")
+        else:
+            logger.info(f"Download cancelled: {task_id}")
+            book_queue.update_status(task_id, QueueStatus.CANCELLED)
+
+        # Broadcast error/cancelled status
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+def concurrent_download_loop() -> None:
+    """Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
+    max_workers = config.MAX_CONCURRENT_DOWNLOADS
+    logger.info(f"Starting concurrent download loop with {max_workers} workers")
+
+    with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor:
+        active_futures: Dict[Future, str] = {}  # Track active download futures
+
+        while True:
+            # Clean up completed futures
+            completed_futures = [f for f in active_futures if f.done()]
+            for future in completed_futures:
+                task_id = active_futures.pop(future)
+                try:
+                    future.result()  # This will raise any exceptions from the worker
+                except Exception as e:
+                    logger.error_trace(f"Future exception for {task_id}: {e}")
+
+            # Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
+            current_time = time.time()
+            with _progress_lock:
+                for future, task_id in list(active_futures.items()):
+                    last_active = _last_activity.get(task_id, current_time)
+                    if current_time - last_active > STALL_TIMEOUT:
+                        logger.warning(f"Download stalled for {task_id}, cancelling")
+                        book_queue.cancel_download(task_id)
+                        book_queue.update_status_message(task_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)")
+
+            # Start new downloads if we have capacity
+            while len(active_futures) < max_workers:
+                next_download = book_queue.get_next()
+                if not next_download:
+                    break
+
+                # Stagger concurrent downloads to avoid rate limiting on shared download servers
+                # Only delay if other downloads are already active
+                if active_futures:
+                    stagger_delay = random.uniform(2, 5)
+                    logger.debug(f"Staggering download start by {stagger_delay:.1f}s")
+                    time.sleep(stagger_delay)
+
+                task_id, cancel_flag = next_download
+
+                # Submit download job to thread pool
+                future = executor.submit(_process_single_download, task_id, cancel_flag)
+                active_futures[future] = task_id
+
+            # Brief sleep to prevent busy waiting
+            time.sleep(config.MAIN_LOOP_SLEEP_TIME)
+
+# Download coordinator thread (started explicitly via start())
+_coordinator_thread: Optional[threading.Thread] = None
+_started = False
+
+
+def start() -> None:
+    """Start the download coordinator thread. Safe to call multiple times."""
+    global _coordinator_thread, _started
+
+    if _started:
+        logger.debug("Download coordinator already started")
+        return
+
+    _coordinator_thread = threading.Thread(
+        target=concurrent_download_loop,
+        daemon=True,
+        name="DownloadCoordinator"
+    )
+    _coordinator_thread.start()
+    _started = True
+
+    logger.info(f"Download coordinator started with {config.MAX_CONCURRENT_DOWNLOADS} concurrent workers")

shelfmark/download/outputs/__init__.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Callable, Optional
+
+from shelfmark.core.models import DownloadTask
+
+StatusCallback = Callable[[str, Optional[str]], None]
+OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback], Optional[str]]
+
+
+@dataclass(frozen=True)
+class OutputRegistration:
+    mode: str
+    supports_task: Callable[[DownloadTask], bool]
+    handler: OutputHandler
+    priority: int = 0
+
+
+_OUTPUT_REGISTRY: list[OutputRegistration] = []
+_OUTPUTS_LOADED = False
+
+
+def register_output(
+    mode: str,
+    supports_task: Callable[[DownloadTask], bool],
+    priority: int = 0,
+) -> Callable[[OutputHandler], OutputHandler]:
+    def decorator(handler: OutputHandler) -> OutputHandler:
+        _OUTPUT_REGISTRY.append(
+            OutputRegistration(
+                mode=mode,
+                supports_task=supports_task,
+                handler=handler,
+                priority=priority,
+            )
+        )
+        _OUTPUT_REGISTRY.sort(key=lambda entry: entry.priority, reverse=True)
+        return handler
+
+    return decorator
+
+
+def load_output_handlers() -> None:
+    global _OUTPUTS_LOADED
+    if _OUTPUTS_LOADED:
+        return
+
+    from . import booklore  # noqa: F401
+    from . import folder  # noqa: F401
+
+    _OUTPUTS_LOADED = True
+
+
+def resolve_output_handler(task: DownloadTask) -> Optional[OutputRegistration]:
+    load_output_handlers()
+    for entry in _OUTPUT_REGISTRY:
+        if entry.supports_task(task):
+            return entry
+    return None

shelfmark/download/outputs/booklore.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Any, Dict, List, Mapping, Optional
+
+import requests
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.outputs import register_output
+from shelfmark.download.staging import STAGE_MOVE, STAGE_NONE, build_staging_dir
+
+logger = setup_logger(__name__)
+
+BOOKLORE_OUTPUT_MODE = "booklore"
+BOOKLORE_SUPPORTED_EXTENSIONS = {".cb7", ".cbr", ".cbz", ".epub", ".fb2", ".pdf"}
+BOOKLORE_SUPPORTED_FORMATS_LABEL = ", ".join(
+    ext.lstrip(".").upper() for ext in sorted(BOOKLORE_SUPPORTED_EXTENSIONS)
+)
+
+
+class BookloreError(Exception):
+    """Raised when Booklore integration fails."""
+
+
+@dataclass(frozen=True)
+class BookloreConfig:
+    base_url: str
+    username: str
+    password: str
+    library_id: int
+    path_id: int
+    verify_tls: bool = True
+    refresh_after_upload: bool = False
+
+
+def _parse_int(value: Any, label: str) -> int:
+    if value is None or value == "":
+        raise BookloreError(f"{label} is required")
+    try:
+        return int(value)
+    except (TypeError, ValueError) as exc:
+        raise BookloreError(f"{label} must be a number") from exc
+
+
+def build_booklore_config(values: Mapping[str, Any]) -> BookloreConfig:
+    base_url = str(values.get("BOOKLORE_HOST", "")).strip()
+    username = str(values.get("BOOKLORE_USERNAME", "")).strip()
+    password = values.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        raise BookloreError("Booklore URL is required")
+    if not username:
+        raise BookloreError("Booklore username is required")
+    if not password:
+        raise BookloreError("Booklore password is required")
+
+    library_id = _parse_int(values.get("BOOKLORE_LIBRARY_ID"), "Booklore library ID")
+    path_id = _parse_int(values.get("BOOKLORE_PATH_ID"), "Booklore path ID")
+
+    return BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=library_id,
+        path_id=path_id,
+        verify_tls=True,
+        refresh_after_upload=True,  # Always refresh library after upload
+    )
+
+
+def booklore_login(booklore_config: BookloreConfig) -> str:
+    url = f"{booklore_config.base_url}/api/v1/auth/login"
+    payload = {"username": booklore_config.username, "password": booklore_config.password}
+
+    try:
+        response = requests.post(url, json=payload, timeout=30, verify=booklore_config.verify_tls)
+    except requests.exceptions.ConnectionError as exc:
+        raise BookloreError("Could not connect to Booklore") from exc
+    except requests.exceptions.Timeout as exc:
+        raise BookloreError("Booklore connection timed out") from exc
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore login failed: {exc}") from exc
+
+    if response.status_code in {401, 403}:
+        raise BookloreError("Booklore authentication failed")
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        raise BookloreError(f"Booklore login failed ({response.status_code})") from exc
+
+    try:
+        data = response.json()
+    except ValueError as exc:
+        raise BookloreError("Invalid Booklore login response") from exc
+
+    token = data.get("accessToken")
+    if not token:
+        raise BookloreError("Booklore did not return an access token")
+
+    return token
+
+
+def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list[dict[str, Any]]:
+    url = f"{booklore_config.base_url}/api/v1/libraries"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Failed to fetch Booklore libraries: {exc}") from exc
+
+    try:
+        return response.json()
+    except ValueError as exc:
+        raise BookloreError("Invalid Booklore libraries response") from exc
+
+
+def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None:
+    url = f"{booklore_config.base_url}/api/v1/files/upload"
+    headers = {"Authorization": f"Bearer {token}"}
+    params = {"libraryId": booklore_config.library_id, "pathId": booklore_config.path_id}
+
+    response = None
+
+    try:
+        with file_path.open("rb") as handle:
+            response = requests.post(
+                url,
+                headers=headers,
+                params=params,
+                files={"file": (file_path.name, handle)},
+                timeout=60,
+                verify=booklore_config.verify_tls,
+            )
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        message = response.text.strip() if response is not None else ""
+        if message:
+            message = f": {message[:200]}"
+        status_code = response.status_code if response is not None else "unknown"
+        raise BookloreError(f"Booklore upload failed ({status_code}){message}") from exc
+    except requests.exceptions.ConnectionError as exc:
+        raise BookloreError("Could not connect to Booklore") from exc
+    except requests.exceptions.Timeout as exc:
+        raise BookloreError("Booklore upload timed out") from exc
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore upload failed: {exc}") from exc
+
+
+def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None:
+    url = f"{booklore_config.base_url}/api/v1/libraries/{booklore_config.library_id}/refresh"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.put(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore refresh failed: {exc}") from exc
+
+
+def _supports_booklore(task: DownloadTask) -> bool:
+    if check_audiobook(task.content_type):
+        return False
+    return core_config.config.get("BOOKS_OUTPUT_MODE", "folder") == BOOKLORE_OUTPUT_MODE
+
+
+def _get_booklore_settings() -> Dict[str, Any]:
+    return {
+        "BOOKLORE_HOST": core_config.config.get("BOOKLORE_HOST", ""),
+        "BOOKLORE_USERNAME": core_config.config.get("BOOKLORE_USERNAME", ""),
+        "BOOKLORE_PASSWORD": core_config.config.get("BOOKLORE_PASSWORD", ""),
+        "BOOKLORE_LIBRARY_ID": core_config.config.get("BOOKLORE_LIBRARY_ID"),
+        "BOOKLORE_PATH_ID": core_config.config.get("BOOKLORE_PATH_ID"),
+    }
+
+
+def _booklore_format_error(rejected_files: List[Path]) -> str:
+    rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+    rejected_list = ", ".join(rejected_exts)
+    return (
+        f"Booklore does not support {rejected_list}. "
+        f"Supported formats: {BOOKLORE_SUPPORTED_FORMATS_LABEL}"
+    )
+
+
+def _post_process_booklore(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    from shelfmark.download.postprocess.pipeline import (
+        OutputPlan,
+        cleanup_output_staging,
+        is_managed_workspace_path,
+        prepare_output_files,
+    )
+
+    if cancel_flag.is_set():
+        logger.info("Task %s: cancelled before Booklore upload", task.task_id)
+        return None
+
+    try:
+        booklore_config = build_booklore_config(_get_booklore_settings())
+    except BookloreError as e:
+        logger.warning("Task %s: Booklore configuration error: %s", task.task_id, e)
+        status_callback("error", str(e))
+        return None
+
+    status_callback("resolving", "Preparing Booklore upload")
+
+    output_plan = OutputPlan(
+        mode=BOOKLORE_OUTPUT_MODE,
+        stage_action=STAGE_MOVE if is_managed_workspace_path(temp_file) else STAGE_NONE,
+        staging_dir=build_staging_dir("booklore", task.task_id),
+        allow_archive_extraction=True,
+    )
+
+    prepared = prepare_output_files(
+        temp_file,
+        task,
+        BOOKLORE_OUTPUT_MODE,
+        status_callback,
+        output_plan=output_plan,
+    )
+    if not prepared:
+        return None
+
+    logger.debug("Task %s: prepared %d file(s) for Booklore upload", task.task_id, len(prepared.files))
+
+    try:
+        unsupported_files = [
+            file_path
+            for file_path in prepared.files
+            if file_path.suffix.lower() not in BOOKLORE_SUPPORTED_EXTENSIONS
+        ]
+        if unsupported_files:
+            error_message = _booklore_format_error(unsupported_files)
+            logger.warning("Task %s: %s", task.task_id, error_message)
+            status_callback("error", error_message)
+            return None
+
+        token = booklore_login(booklore_config)
+        logger.info("Task %s: uploading %d file(s) to Booklore", task.task_id, len(prepared.files))
+
+        for index, file_path in enumerate(prepared.files, start=1):
+            if cancel_flag.is_set():
+                logger.info("Task %s: cancelled during Booklore upload", task.task_id)
+                return None
+            status_callback("resolving", f"Uploading to Booklore ({index}/{len(prepared.files)})")
+            booklore_upload_file(booklore_config, token, file_path)
+
+        if booklore_config.refresh_after_upload:
+            try:
+                booklore_refresh_library(booklore_config, token)
+            except BookloreError as e:
+                logger.warning("Task %s: Booklore refresh failed: %s", task.task_id, e)
+
+        logger.info("Task %s: uploaded %d file(s) to Booklore", task.task_id, len(prepared.files))
+
+        message = "Uploaded to Booklore"
+        if len(prepared.files) > 1:
+            message = f"Uploaded to Booklore ({len(prepared.files)} files)"
+        status_callback("complete", message)
+        return f"booklore://{task.task_id}"
+
+    except BookloreError as e:
+        logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e)
+        status_callback("error", str(e))
+        return None
+    except Exception as e:
+        logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e)
+        status_callback("error", f"Booklore upload failed: {e}")
+        return None
+    finally:
+        cleanup_output_staging(
+            prepared.output_plan,
+            prepared.working_path,
+            task,
+            prepared.cleanup_paths,
+        )
+
+
+@register_output(BOOKLORE_OUTPUT_MODE, supports_task=_supports_booklore, priority=10)
+def process_booklore_output(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    return _post_process_booklore(temp_file, task, cancel_flag, status_callback)

shelfmark/download/outputs/folder.py

@@ -0,0 +1,310 @@
+from __future__ import annotations
+
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Any, Optional, List
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.archive import is_archive
+from shelfmark.download.outputs import register_output
+from shelfmark.download.staging import StageAction, STAGE_NONE
+
+logger = setup_logger(__name__)
+
+FOLDER_OUTPUT_MODE = "folder"
+
+
+def _resolve_custom_script_target(target_path: Path, destination: Path, path_mode: str) -> Path:
+    mode = (path_mode or "absolute").strip().lower()
+    if mode != "relative":
+        return target_path
+
+    try:
+        return target_path.relative_to(destination)
+    except ValueError:
+        if target_path.is_absolute():
+            return Path(target_path.name)
+        return target_path
+
+
+@dataclass(frozen=True)
+class _ProcessingPlan:
+    destination: Path
+    organization_mode: str
+    use_hardlink: bool
+    allow_archive_extraction: bool
+    stage_action: StageAction
+    staging_dir: Path
+    hardlink_source: Optional[Path]
+    output_mode: str = FOLDER_OUTPUT_MODE
+
+
+def _supports_folder_output(task: DownloadTask) -> bool:
+    if check_audiobook(task.content_type):
+        return True
+    return core_config.config.get("BOOKS_OUTPUT_MODE", FOLDER_OUTPUT_MODE) == FOLDER_OUTPUT_MODE
+
+
+def _build_processing_plan(
+    temp_file: Path,
+    task: DownloadTask,
+    status_callback,
+) -> Optional[_ProcessingPlan]:
+    from shelfmark.download.postprocess.pipeline import (
+        build_output_plan,
+        get_final_destination,
+        validate_destination,
+    )
+    from shelfmark.download.postprocess.policy import get_file_organization
+
+    is_audiobook = check_audiobook(task.content_type)
+    organization_mode = get_file_organization(is_audiobook)
+    destination = get_final_destination(task)
+
+    if not validate_destination(destination, status_callback):
+        return None
+
+    output_plan = build_output_plan(
+        temp_file,
+        task,
+        output_mode=FOLDER_OUTPUT_MODE,
+        destination=destination,
+        status_callback=status_callback,
+    )
+    if not output_plan.transfer_plan:
+        return None
+
+    transfer_plan = output_plan.transfer_plan
+    hardlink_source = transfer_plan.source_path if transfer_plan.use_hardlink else None
+
+    return _ProcessingPlan(
+        destination=destination,
+        organization_mode=organization_mode,
+        use_hardlink=transfer_plan.use_hardlink,
+        allow_archive_extraction=transfer_plan.allow_archive_extraction,
+        stage_action=output_plan.stage_action,
+        staging_dir=output_plan.staging_dir,
+        hardlink_source=hardlink_source,
+    )
+
+
+@register_output(FOLDER_OUTPUT_MODE, supports_task=_supports_folder_output, priority=0)
+def process_folder_output(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    """Post-process download to the configured folder destination."""
+    from shelfmark.download.postprocess.pipeline import (
+        cleanup_output_staging,
+        is_torrent_source,
+        log_plan_steps,
+        prepare_output_files,
+        record_step,
+        safe_cleanup_path,
+        transfer_book_files,
+    )
+
+    plan = _build_processing_plan(temp_file, task, status_callback)
+    if not plan:
+        return None
+
+    logger.debug(
+        "Processing plan for task %s: mode=%s destination=%s hardlink=%s stage_action=%s extract_archives=%s",
+        task.task_id,
+        plan.organization_mode,
+        plan.destination,
+        plan.use_hardlink,
+        plan.stage_action,
+        plan.allow_archive_extraction,
+    )
+
+    prepared = prepare_output_files(
+        temp_file,
+        task,
+        output_mode=plan.output_mode,
+        status_callback=status_callback,
+        destination=plan.destination,
+    )
+    if not prepared:
+        return None
+
+    steps: List[Any] = []
+    if prepared.output_plan.stage_action != STAGE_NONE:
+        step_name = f"stage_{prepared.output_plan.stage_action}"
+        record_step(steps, step_name, source=str(temp_file), dest=str(prepared.output_plan.staging_dir))
+
+    def run_custom_script(script_path: str, target_path: Path, phase: str) -> bool:
+        path_mode = core_config.config.get("CUSTOM_SCRIPT_PATH_MODE", "absolute")
+        script_target = _resolve_custom_script_target(target_path, plan.destination, path_mode)
+        env = {
+            **os.environ,
+            "SHELFMARK_CUSTOM_SCRIPT_TARGET": str(target_path),
+            "SHELFMARK_CUSTOM_SCRIPT_RELATIVE": str(_resolve_custom_script_target(target_path, plan.destination, "relative")),
+            "SHELFMARK_CUSTOM_SCRIPT_DESTINATION": str(plan.destination),
+            "SHELFMARK_CUSTOM_SCRIPT_MODE": str(path_mode),
+            "SHELFMARK_CUSTOM_SCRIPT_PHASE": phase,
+        }
+        record_step(
+            steps,
+            "custom_script",
+            script=str(script_path),
+            target=str(script_target),
+            target_abs=str(target_path),
+            mode=str(path_mode),
+            phase=phase,
+        )
+        log_plan_steps(task.task_id, steps)
+        logger.info(
+            "Task %s: running custom script %s on %s (%s)",
+            task.task_id,
+            script_path,
+            script_target,
+            phase,
+        )
+        try:
+            result = subprocess.run(
+                [script_path, str(script_target)],
+                check=True,
+                timeout=300,  # 5 minute timeout
+                capture_output=True,
+                text=True,
+                env=env,
+            )
+            if result.stdout:
+                logger.debug("Task %s: custom script stdout: %s", task.task_id, result.stdout.strip())
+            return True
+        except FileNotFoundError:
+            logger.error("Task %s: custom script not found: %s", task.task_id, script_path)
+            status_callback("error", f"Custom script not found: {script_path}")
+            return False
+        except PermissionError:
+            logger.error("Task %s: custom script not executable: %s", task.task_id, script_path)
+            status_callback("error", f"Custom script not executable: {script_path}")
+            return False
+        except subprocess.TimeoutExpired:
+            logger.error("Task %s: custom script timed out after 300s: %s", task.task_id, script_path)
+            status_callback("error", "Custom script timed out")
+            return False
+        except subprocess.CalledProcessError as e:
+            stderr = e.stderr.strip() if e.stderr else "No error output"
+            logger.error(
+                "Task %s: custom script failed (exit code %s): %s",
+                task.task_id,
+                e.returncode,
+                stderr,
+            )
+            status_callback("error", f"Custom script failed: {stderr[:100]}")
+            return False
+
+    # Custom script is run post-transfer (see below).
+
+    # If we staged a copy into TMP_DIR (e.g. for custom script), transfer from the staged
+    # path and disable hardlinking for this transfer.
+    use_hardlink = plan.use_hardlink and prepared.output_plan.stage_action == STAGE_NONE
+    source_path = plan.hardlink_source if use_hardlink and plan.hardlink_source else prepared.working_path
+    is_torrent = is_torrent_source(source_path, task)
+
+    usenet_action = core_config.config.get("PROWLARR_USENET_ACTION", "move")
+    is_usenet = task.source == "prowlarr" and not task.original_download_path
+
+    # For external usenet downloads, always copy from the client path.
+    # "Move" is implemented as a client-side cleanup after import.
+    preserve_source = is_usenet
+
+    copy_for_label = is_torrent or preserve_source or prepared.output_plan.stage_action != STAGE_NONE
+
+    if cancel_flag.is_set():
+        logger.info("Task %s: cancelled before final transfer", task.task_id)
+        cleanup_output_staging(
+            prepared.output_plan,
+            prepared.working_path,
+            task,
+            prepared.cleanup_paths,
+        )
+        return None
+
+    if use_hardlink:
+        op_label = "Hardlinking"
+    elif is_usenet and usenet_action == "move" and prepared.output_plan.stage_action == STAGE_NONE:
+        # Presented as a move, but implemented as copy + client cleanup.
+        op_label = "Moving"
+    elif copy_for_label:
+        op_label = "Copying"
+    else:
+        op_label = "Moving"
+
+    status_callback("resolving", f"{op_label} file")
+    record_step(
+        steps,
+        "transfer",
+        op=op_label.lower(),
+        source=str(source_path),
+        dest=str(plan.destination),
+        hardlink=use_hardlink,
+        torrent=copy_for_label,
+    )
+    if prepared.output_plan.stage_action != STAGE_NONE:
+        record_step(steps, "cleanup_staging", path=str(prepared.working_path))
+    log_plan_steps(task.task_id, steps)
+
+    final_paths, error = transfer_book_files(
+        prepared.files,
+        destination=plan.destination,
+        task=task,
+        use_hardlink=use_hardlink,
+        is_torrent=is_torrent,
+        preserve_source=preserve_source,
+        organization_mode=plan.organization_mode,
+    )
+
+    if error:
+        logger.warning("Task %s: transfer failed: %s", task.task_id, error)
+        status_callback("error", error)
+        return None
+
+    logger.info(
+        "Task %s: transferred %d file(s) to %s (%s)",
+        task.task_id,
+        len(final_paths),
+        plan.destination,
+        op_label.lower(),
+    )
+
+    # Run custom script once per successful task, after transfer.
+    if core_config.config.CUSTOM_SCRIPT:
+        if len(final_paths) == 1:
+            target_path = final_paths[0]
+        else:
+            try:
+                target_path = Path(os.path.commonpath([str(p.parent) for p in final_paths]))
+            except ValueError:
+                target_path = plan.destination
+
+        if not run_custom_script(core_config.config.CUSTOM_SCRIPT, target_path, phase="post_transfer"):
+            cleanup_output_staging(
+                prepared.output_plan,
+                prepared.working_path,
+                task,
+                prepared.cleanup_paths,
+            )
+            return None
+
+    cleanup_output_staging(
+        prepared.output_plan,
+        prepared.working_path,
+        task,
+        prepared.cleanup_paths,
+    )
+
+    message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
+    status_callback("complete", message)
+
+    return str(final_paths[0])

shelfmark/download/permissions_debug.py

@@ -0,0 +1,125 @@
+"""Permission/ownership diagnostics for filesystem operations.
+
+This module centralizes best-effort debug logging used by download post-processing
+and atomic filesystem operations.
+
+It is intentionally defensive: failures collecting context should never mask the
+original error.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def _format_uid(uid: int) -> str:
+    try:
+        import pwd
+
+        return pwd.getpwuid(uid).pw_name
+    except Exception:
+        return str(uid)
+
+
+def _format_gid(gid: int) -> str:
+    try:
+        import grp
+
+        return grp.getgrgid(gid).gr_name
+    except Exception:
+        return str(gid)
+
+
+def log_path_permission_context(label: str, path: Path) -> None:
+    """Log useful permission/ownership context for a path.
+
+    Only call this from failure paths.
+    """
+
+    try:
+        euid = os.geteuid() if hasattr(os, "geteuid") else None
+        egid = os.getegid() if hasattr(os, "getegid") else None
+        groups = os.getgroups() if hasattr(os, "getgroups") else []
+
+        if euid is not None and egid is not None:
+            logger.debug(
+                "Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s",
+                label,
+                _format_uid(euid),
+                euid,
+                _format_gid(egid),
+                egid,
+                [f"{_format_gid(g)}({g})" for g in groups],
+            )
+
+        for probe in [path, path.parent]:
+            try:
+                resolved = probe.resolve()
+            except Exception:
+                resolved = probe
+
+            try:
+                st = probe.stat()
+                logger.debug(
+                    "Path permissions (%s): path=%s resolved=%s mode=%s owner=%s(%d) group=%s(%d) dir=%s symlink=%s",
+                    label,
+                    probe,
+                    resolved,
+                    oct(st.st_mode & 0o777),
+                    _format_uid(st.st_uid),
+                    st.st_uid,
+                    _format_gid(st.st_gid),
+                    st.st_gid,
+                    probe.is_dir(),
+                    probe.is_symlink(),
+                )
+            except Exception as stat_error:
+                logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
+    except Exception as context_error:
+        logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
+
+
+def log_transfer_permission_context(label: str, source: Path, dest: Path, error: Exception) -> None:
+    """Log useful permission/ownership context when a file transfer fails."""
+
+    try:
+        euid = os.geteuid() if hasattr(os, "geteuid") else None
+        egid = os.getegid() if hasattr(os, "getegid") else None
+        groups = os.getgroups() if hasattr(os, "getgroups") else []
+
+        if euid is not None and egid is not None:
+            logger.debug(
+                "Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s error=%s",
+                label,
+                _format_uid(euid),
+                euid,
+                _format_gid(egid),
+                egid,
+                [f"{_format_gid(g)}({g})" for g in groups],
+                error,
+            )
+
+        for probe in [source, dest, dest.parent]:
+            try:
+                st = probe.stat()
+                logger.debug(
+                    "Path permissions (%s): path=%s mode=%s owner=%s(%d) group=%s(%d) exists=%s dir=%s",
+                    label,
+                    probe,
+                    oct(st.st_mode & 0o777),
+                    _format_uid(st.st_uid),
+                    st.st_uid,
+                    _format_gid(st.st_gid),
+                    st.st_gid,
+                    probe.exists(),
+                    probe.is_dir(),
+                )
+            except Exception as stat_error:
+                logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
+    except Exception as context_error:
+        logger.debug("Permission context (%s): failed to collect: %s", label, context_error)

shelfmark/download/postprocess/__init__.py

@@ -0,0 +1,11 @@
+"""Post-download processing pipeline.
+
+This package contains the post-download processing pipeline (staging, scanning,
+archive extraction, transfers, and safe cleanup) and the router that selects an
+output handler.
+
+Output handlers live in `shelfmark.download.outputs` and should depend on
+`pipeline` (not `router`) to avoid circular imports.
+"""
+
+from .router import post_process_download

shelfmark/download/postprocess/destination.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import (
+    get_aa_content_type_dir,
+    get_destination,
+    is_audiobook as check_audiobook,
+)
+from shelfmark.download.permissions_debug import log_path_permission_context
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def validate_destination(destination: Path, status_callback) -> bool:
+    """Validate destination path is absolute, exists, and writable."""
+
+    if not destination.is_absolute():
+        logger.warning(f"Destination must be absolute: {destination}")
+        status_callback("error", f"Destination must be absolute: {destination}")
+        return False
+
+    if destination.exists() and not destination.is_dir():
+        logger.warning(f"Destination is not a directory: {destination}")
+        status_callback("error", f"Destination is not a directory: {destination}")
+        return False
+
+    if not destination.exists():
+        try:
+            destination.mkdir(parents=True, exist_ok=True)
+        except (OSError, PermissionError) as exc:
+            log_path_permission_context("destination_create", destination)
+            logger.warning(f"Cannot create destination: {destination} ({exc})")
+            status_callback("error", f"Cannot create destination: {destination} ({exc})")
+            return False
+
+    test_path = destination / f".shelfmark_write_test_{uuid.uuid4().hex}.tmp"
+
+    try:
+        test_content = (
+            f"This file was created to verify if '{destination}' is writable. "
+            "It should've been automatically deleted. Feel free to delete it.\n"
+        )
+        test_path.write_text(test_content)
+        test_path.unlink(missing_ok=True)
+    except Exception as exc:
+        logger.debug("Destination write probe path: %s", test_path)
+        log_path_permission_context("destination_write_probe", destination)
+        logger.warning(f"Destination not writable: {destination} ({exc})")
+        status_callback("error", f"Destination not writable: {destination} ({exc})")
+        return False
+
+    return True
+
+
+def get_final_destination(task: DownloadTask) -> Path:
+    """Get final destination directory, with content-type routing support."""
+
+    is_audiobook = check_audiobook(task.content_type)
+
+    if task.source == "direct_download" and not is_audiobook:
+        override = get_aa_content_type_dir(task.content_type)
+        if override:
+            return override
+
+    return get_destination(is_audiobook)

shelfmark/download/postprocess/pipeline.py

@@ -0,0 +1,76 @@
+"""Post-download processing pipeline.
+
+This module is the public API surface for post-download processing.
+
+Implementation lives in submodules in this package:
+
+- `types`: dataclasses used across the pipeline
+- `workspace`: managed workspace + cleanup rules
+- `scan`: directory scanning + archive extraction
+- `transfer`: hardlink/copy/move + naming/organization
+- `prepare`: staging plan + prepared file selection
+- `steps`: lightweight plan logging helpers
+
+Keeping this file as a facade avoids churn in call sites while letting the
+implementation stay modular.
+"""
+
+from __future__ import annotations
+
+from .destination import get_final_destination, validate_destination
+from .prepare import build_output_plan, prepare_output_files
+from .scan import (
+    collect_directory_files,
+    collect_staged_files,
+    extract_archive_files,
+    get_supported_formats,
+    scan_directory_tree,
+)
+from .steps import log_plan_steps, record_step
+from .transfer import (
+    build_metadata_dict,
+    is_torrent_source,
+    process_directory,
+    resolve_hardlink_source,
+    should_hardlink,
+    transfer_book_files,
+    transfer_directory_to_library,
+    transfer_file_to_library,
+)
+from .types import OutputPlan, PlanStep, PreparedFiles, TransferPlan
+from .workspace import (
+    cleanup_output_staging,
+    is_managed_workspace_path,
+    is_within_tmp_dir,
+    safe_cleanup_path,
+)
+
+__all__ = [
+    "OutputPlan",
+    "PlanStep",
+    "PreparedFiles",
+    "TransferPlan",
+    "build_metadata_dict",
+    "build_output_plan",
+    "cleanup_output_staging",
+    "collect_directory_files",
+    "collect_staged_files",
+    "extract_archive_files",
+    "get_final_destination",
+    "get_supported_formats",
+    "is_managed_workspace_path",
+    "is_torrent_source",
+    "is_within_tmp_dir",
+    "log_plan_steps",
+    "prepare_output_files",
+    "process_directory",
+    "record_step",
+    "resolve_hardlink_source",
+    "safe_cleanup_path",
+    "scan_directory_tree",
+    "should_hardlink",
+    "transfer_book_files",
+    "transfer_directory_to_library",
+    "transfer_file_to_library",
+    "validate_destination",
+]

shelfmark/download/postprocess/policy.py

@@ -0,0 +1,100 @@
+"""Post-download processing policy.
+
+This module holds configuration-driven *policy* decisions that are shared across
+post-download processing components, but are not specific to archive extraction.
+
+Examples:
+- Which file formats are enabled
+- How files should be organized (none/rename/organize)
+- Which naming templates to use
+
+Implementation note:
+Keep this module free of dependencies on archive extraction mechanics to avoid
+circular imports (`archive` is used by the pipeline).
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+import shelfmark.core.config as core_config
+
+
+def get_supported_formats() -> List[str]:
+    """Get current supported formats from config singleton."""
+
+    formats = core_config.config.get(
+        "SUPPORTED_FORMATS",
+        ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
+    )
+
+    # Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
+    if isinstance(formats, str):
+        return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
+
+    return [fmt.lower() for fmt in formats]
+
+
+def get_supported_audiobook_formats() -> List[str]:
+    """Get current supported audiobook formats from config singleton."""
+
+    formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
+
+    # Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
+    if isinstance(formats, str):
+        return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
+
+    return [fmt.lower() for fmt in formats]
+
+
+def get_file_organization(is_audiobook: bool) -> str:
+    """Get the file organization mode for the content type."""
+
+    key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
+    mode = core_config.config.get(key, "rename")
+
+    # Handle legacy settings migration
+    if mode not in ("none", "rename", "organize"):
+        legacy_key = "PROCESSING_MODE_AUDIOBOOK" if is_audiobook else "PROCESSING_MODE"
+        legacy_mode = core_config.config.get(legacy_key, "ingest")
+        if legacy_mode == "library":
+            return "organize"
+        if core_config.config.get("USE_BOOK_TITLE", True):
+            return "rename"
+        return "none"
+
+    return mode
+
+
+def get_template(is_audiobook: bool, organization_mode: str) -> str:
+    """Get the template for the content type and organization mode."""
+
+    # Determine the correct key based on content type and organization mode
+    if is_audiobook:
+        if organization_mode == "organize":
+            key = "TEMPLATE_AUDIOBOOK_ORGANIZE"
+        else:
+            key = "TEMPLATE_AUDIOBOOK_RENAME"
+    else:
+        if organization_mode == "organize":
+            key = "TEMPLATE_ORGANIZE"
+        else:
+            key = "TEMPLATE_RENAME"
+
+    template = core_config.config.get(key, "")
+
+    # Fallback to legacy keys if new keys are empty
+    if not template:
+        legacy_key = "TEMPLATE_AUDIOBOOK" if is_audiobook else "TEMPLATE"
+        template = core_config.config.get(legacy_key, "")
+
+    if not template:
+        legacy_key = "LIBRARY_TEMPLATE_AUDIOBOOK" if is_audiobook else "LIBRARY_TEMPLATE"
+        template = core_config.config.get(legacy_key, "")
+
+    if not template:
+        if organization_mode == "organize":
+            return "{Author}/{Title} ({Year})"
+        return "{Author} - {Title} ({Year})"
+
+    return template

shelfmark/download/postprocess/prepare.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.download.archive import is_archive
+from shelfmark.download.staging import STAGE_COPY, STAGE_NONE, get_staging_dir, stage_path
+
+from .scan import collect_staged_files
+from .transfer import resolve_hardlink_source
+from .types import OutputPlan, PreparedFiles
+from .workspace import cleanup_output_staging, is_managed_workspace_path
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def build_output_plan(
+    temp_file: Path,
+    task: DownloadTask,
+    output_mode: str,
+    destination: Optional[Path] = None,
+    status_callback=None,
+) -> OutputPlan:
+    """Build an output plan that describes staging behavior for file-based outputs."""
+
+    transfer_plan = resolve_hardlink_source(temp_file, task, destination, status_callback)
+    runs_custom_script = bool(core_config.config.CUSTOM_SCRIPT) and temp_file.is_file() and not is_archive(temp_file)
+
+    stage_action = STAGE_COPY if runs_custom_script and not is_managed_workspace_path(temp_file) else STAGE_NONE
+    staging_dir = get_staging_dir()
+
+    return OutputPlan(
+        mode=output_mode,
+        stage_action=stage_action,
+        staging_dir=staging_dir,
+        allow_archive_extraction=transfer_plan.allow_archive_extraction,
+        transfer_plan=transfer_plan,
+    )
+
+
+def prepare_output_files(
+    temp_file: Path,
+    task: DownloadTask,
+    output_mode: str,
+    status_callback,
+    destination: Optional[Path] = None,
+    output_plan: Optional[OutputPlan] = None,
+) -> Optional[PreparedFiles]:
+    if output_plan is None:
+        output_plan = build_output_plan(
+            temp_file,
+            task,
+            output_mode=output_mode,
+            destination=destination,
+            status_callback=status_callback,
+        )
+
+    working_path = temp_file
+    if output_plan.stage_action != STAGE_NONE:
+        step_label = "Staging torrent files" if output_plan.stage_action == STAGE_COPY else "Staging files"
+        status_callback("resolving", step_label)
+        working_path = stage_path(working_path, output_plan.staging_dir, output_plan.stage_action)
+
+    can_delete_source_archives = output_plan.stage_action != STAGE_NONE or is_managed_workspace_path(working_path)
+
+    files, rejected_files, cleanup_paths, error = collect_staged_files(
+        working_path=working_path,
+        task=task,
+        allow_archive_extraction=output_plan.allow_archive_extraction,
+        status_callback=status_callback,
+        cleanup_archives=can_delete_source_archives,
+    )
+
+    if error:
+        status_callback("error", error)
+        cleanup_output_staging(output_plan, working_path, task, cleanup_paths)
+        return None
+
+    if output_plan.stage_action == STAGE_NONE and is_managed_workspace_path(working_path):
+        cleanup_paths = [*cleanup_paths, working_path]
+
+    return PreparedFiles(
+        output_plan=output_plan,
+        working_path=working_path,
+        files=files,
+        rejected_files=rejected_files,
+        cleanup_paths=cleanup_paths,
+    )

shelfmark/download/postprocess/router.py

@@ -0,0 +1,52 @@
+"""Output routing for post-download processing.
+
+This module selects the appropriate output handler and invokes it.
+
+Keeping this separate from `pipeline.py` avoids circular imports:
+
+- output handlers depend on `pipeline`
+- router depends on the output registry
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from threading import Event
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask, SearchMode
+from shelfmark.download.outputs import resolve_output_handler
+
+logger = setup_logger(__name__)
+
+
+def post_process_download(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    """Post-process download using the selected output handler."""
+
+    if task.search_mode is None:
+        logger.warning(
+            "Task %s: missing search_mode; defaulting to Direct mode behavior",
+            task.task_id,
+        )
+    elif task.search_mode not in (SearchMode.DIRECT, SearchMode.UNIVERSAL):
+        logger.warning(
+            "Task %s: invalid search_mode=%s; defaulting to Direct mode behavior",
+            task.task_id,
+            task.search_mode,
+        )
+
+    output_handler = resolve_output_handler(task)
+    if output_handler:
+        logger.info("Task %s: using output mode %s", task.task_id, output_handler.mode)
+        return output_handler.handler(temp_file, task, cancel_flag, status_callback)
+
+    from shelfmark.download.outputs.folder import process_folder_output
+
+    logger.info("Task %s: using output mode folder", task.task_id)
+    return process_folder_output(temp_file, task, cancel_flag, status_callback)

shelfmark/download/postprocess/scan.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
+from shelfmark.download.permissions_debug import log_path_permission_context
+from shelfmark.download.postprocess.policy import (
+    get_supported_audiobook_formats,
+    get_supported_formats as get_book_formats,
+)
+from shelfmark.download.staging import build_staging_dir
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def get_supported_formats(content_type: Optional[str] = None) -> List[str]:
+    if check_audiobook(content_type):
+        return get_supported_audiobook_formats()
+    return get_book_formats()
+
+
+def _format_not_supported_error(rejected_files: List[Path], task: DownloadTask) -> str:
+    content_type = task.content_type
+    file_type_label = "audiobook" if check_audiobook(content_type) else "book"
+    rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+    rejected_list = ", ".join(rejected_exts)
+    supported_formats = get_supported_formats(content_type)
+
+    logger.warning(
+        "Task %s: found %d %s(s) but none match supported formats. Rejected formats: %s. Supported: %s",
+        task.task_id,
+        len(rejected_files),
+        file_type_label,
+        rejected_list,
+        ", ".join(sorted(supported_formats)),
+    )
+
+    return (
+        f"Found {len(rejected_files)} {file_type_label}(s) but format not supported ({rejected_list}). "
+        "Enable in Settings > Formats."
+    )
+
+
+def extract_archive_files(
+    archive_path: Path,
+    output_dir: Path,
+    task: DownloadTask,
+    cleanup_archive: bool,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    content_type = task.content_type
+
+    try:
+        extracted_files, warnings, rejected_files = extract_archive(archive_path, output_dir, content_type)
+    except ArchiveExtractionError as exc:
+        logger.warning(
+            "Task %s: archive extraction failed for %s: %s",
+            task.task_id,
+            archive_path.name,
+            exc,
+        )
+        return [], [], [], str(exc)
+
+    if warnings:
+        logger.debug(
+            "Task %s: archive warnings for %s: %s",
+            task.task_id,
+            archive_path.name,
+            "; ".join(warnings),
+        )
+
+    if cleanup_archive:
+        archive_path.unlink(missing_ok=True)
+
+    cleanup_paths = [output_dir]
+
+    if not extracted_files:
+        if rejected_files:
+            return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
+        file_type_label = "audiobook" if check_audiobook(content_type) else "book"
+        return [], rejected_files, cleanup_paths, f"No {file_type_label} files found in archive"
+
+    logger.debug(
+        "Task %s: extracted %d file(s) from archive %s",
+        task.task_id,
+        len(extracted_files),
+        archive_path.name,
+    )
+
+    return extracted_files, rejected_files, cleanup_paths, None
+
+
+def scan_directory_tree(
+    directory: Path,
+    content_type: Optional[str],
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    """Scan a directory tree for book files, trackable-but-unsupported files, and archives."""
+
+    try:
+        with os.scandir(directory) as it:
+            next(it, None)
+    except PermissionError as exc:
+        log_path_permission_context("scan_directory", directory)
+        logger.warning(f"Permission denied scanning directory: {directory} ({exc})")
+        return [], [], [], f"Permission denied accessing download folder: {directory}"
+    except (FileNotFoundError, NotADirectoryError, OSError) as exc:
+        logger.warning(f"Cannot access download folder: {directory} ({exc})")
+        return [], [], [], f"Cannot access download folder: {directory} ({exc})"
+
+    book_files: List[Path] = []
+    rejected_files: List[Path] = []
+    archive_files: List[Path] = []
+
+    supported_formats = get_supported_formats(content_type)
+    supported_exts = {f".{fmt}" for fmt in supported_formats}
+
+    is_audiobook = check_audiobook(content_type)
+    if is_audiobook:
+        trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
+    else:
+        trackable_exts = {
+            '.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
+            '.doc', '.docx', '.rtf', '.txt',
+        }
+
+    logged_walk_permission_context = False
+
+    def onerror(error: OSError) -> None:
+        nonlocal logged_walk_permission_context
+
+        if isinstance(error, PermissionError):
+            if not logged_walk_permission_context:
+                try:
+                    error_path = Path(getattr(error, "filename", "") or str(directory))
+                except Exception:
+                    error_path = directory
+
+                log_path_permission_context("scan_directory_walk", error_path)
+                logged_walk_permission_context = True
+
+            logger.debug(f"Skipping inaccessible path during scan: {error}")
+        else:
+            logger.debug(f"Error scanning directory tree: {error}")
+
+    for root, _, files in os.walk(directory, onerror=onerror):
+        for filename in files:
+            file_path = Path(root) / filename
+            suffix = file_path.suffix.lower()
+
+            if suffix in supported_exts:
+                book_files.append(file_path)
+            elif suffix in trackable_exts:
+                rejected_files.append(file_path)
+
+            if is_archive(file_path):
+                archive_files.append(file_path)
+
+    return book_files, rejected_files, archive_files, None
+
+
+def collect_directory_files(
+    directory: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool,
+    status_callback=None,
+    cleanup_archives: bool = False,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    content_type = task.content_type
+    book_files, rejected_files, archive_files, scan_error = scan_directory_tree(directory, content_type)
+    if scan_error:
+        return [], [], [], scan_error
+
+    if book_files:
+        if archive_files:
+            logger.debug(
+                "Task %s: ignoring %d archive(s) - already have %d book file(s)",
+                task.task_id,
+                len(archive_files),
+                len(book_files),
+            )
+        if rejected_files:
+            rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+            logger.debug(
+                "Task %s: also found %d file(s) with unsupported formats: %s",
+                task.task_id,
+                len(rejected_files),
+                ", ".join(rejected_exts),
+            )
+        return book_files, rejected_files, [], None
+
+    if archive_files:
+        if not allow_archive_extraction:
+            logger.warning(
+                "Task %s: archive extraction disabled (torrent hardlinking enabled) for %s",
+                task.task_id,
+                directory,
+            )
+            return [], rejected_files, [], "Archive extraction is disabled when torrent hardlinking is enabled"
+
+        if status_callback:
+            status_callback("resolving", "Extracting archives")
+
+        logger.info("Task %s: extracting %d archive(s)", task.task_id, len(archive_files))
+
+        all_files: List[Path] = []
+        all_errors: List[str] = []
+        cleanup_paths: List[Path] = []
+
+        for archive in archive_files:
+            extract_dir = build_staging_dir("extract", task.task_id)
+            extracted_files, archive_rejected, archive_cleanup, error = extract_archive_files(
+                archive_path=archive,
+                output_dir=extract_dir,
+                task=task,
+                cleanup_archive=cleanup_archives,
+            )
+
+            if error:
+                all_errors.append(f"{archive.name}: {error}")
+            if archive_rejected:
+                rejected_files.extend(archive_rejected)
+            if extracted_files:
+                all_files.extend(extracted_files)
+            if archive_cleanup:
+                cleanup_paths.extend(archive_cleanup)
+
+        if all_files:
+            logger.info(
+                "Task %s: extracted %d file(s) from %d archive(s)",
+                task.task_id,
+                len(all_files),
+                len(archive_files),
+            )
+            return all_files, rejected_files, cleanup_paths, None
+
+        if all_errors:
+            return [], rejected_files, cleanup_paths, "; ".join(all_errors)
+
+        if rejected_files:
+            return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
+
+        return [], rejected_files, cleanup_paths, "No book files found in archives"
+
+    if rejected_files:
+        return [], rejected_files, [], _format_not_supported_error(rejected_files, task)
+
+    return [], rejected_files, [], "No book files found in download"
+
+
+def collect_staged_files(
+    working_path: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool,
+    status_callback,
+    cleanup_archives: bool,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    if working_path.is_dir():
+        if status_callback:
+            status_callback("resolving", "Processing download folder")
+        return collect_directory_files(
+            working_path,
+            task,
+            allow_archive_extraction=allow_archive_extraction,
+            status_callback=status_callback,
+            cleanup_archives=cleanup_archives,
+        )
+
+    if is_archive(working_path) and allow_archive_extraction:
+        if status_callback:
+            status_callback("resolving", "Extracting archive")
+
+        logger.info("Task %s: extracting archive %s", task.task_id, working_path.name)
+
+        extract_dir = build_staging_dir("extract", task.task_id)
+        extracted_files, rejected_files, cleanup_paths, error = extract_archive_files(
+            archive_path=working_path,
+            output_dir=extract_dir,
+            task=task,
+            cleanup_archive=cleanup_archives,
+        )
+
+        if extracted_files:
+            logger.info(
+                "Task %s: extracted %d file(s) from archive %s",
+                task.task_id,
+                len(extracted_files),
+                working_path.name,
+            )
+
+        return extracted_files, rejected_files, cleanup_paths, error
+
+    # Single-file download result (non-archive).
+    # Ensure we respect the user's supported format settings.
+    suffix = working_path.suffix.lower()
+    supported_formats = get_supported_formats(task.content_type)
+    supported_exts = {f".{fmt}" for fmt in supported_formats}
+
+    is_audiobook = check_audiobook(task.content_type)
+    if is_audiobook:
+        trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
+    else:
+        trackable_exts = {
+            '.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
+            '.doc', '.docx', '.rtf', '.txt',
+        }
+
+    if suffix in supported_exts:
+        return [working_path], [], [], None
+
+    if suffix in trackable_exts:
+        return [], [working_path], [], _format_not_supported_error([working_path], task)
+
+    file_type_label = "audiobook" if is_audiobook else "book"
+    return [], [], [], f"Unsupported {file_type_label} file type: {suffix or working_path.name}"

shelfmark/download/postprocess/steps.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import Any, List
+
+from shelfmark.core.logger import setup_logger
+
+from .types import PlanStep
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def record_step(steps: List[PlanStep], name: str, **details: Any) -> None:
+    steps.append(PlanStep(name=name, details=details))
+
+
+def log_plan_steps(task_id: str, steps: List[PlanStep]) -> None:
+    if not steps:
+        return
+    summary = " -> ".join(step.name for step in steps)
+    logger.debug("Processing plan for %s: %s", task_id, summary)