Fix: HTTP grab behavior, logging enforcement (#521)

- Base URL option in settings for reverse proxy setups
- Fix NZB downloads not deleting on completion
- Fix handling for audiobook files over 100+ parts
- Fix prowlarr search timeout 
- Fix prowlarr categorisation for expanded searches

.github/workflows/build-and-publish-docker-image.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository_owner }}/shelfmark
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -20,15 +20,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - suffix: ""
-            target: cwa-bd
-            image_name_suffix: ""
-          - suffix: "-tor"
-            target: cwa-bd-tor
-            image_name_suffix: "-tor"
-          - suffix: "-extbp"
-            target: cwa-bd-extbp
-            image_name_suffix: "-extbp"
+          - target: shelfmark
+          - target: shelfmark-lite
+            image_name_suffix: "-lite"
     steps:
       - name: Get current date
         id: date
@@ -80,4 +74,71 @@ jobs:
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
+
+  # Create aliases for backwards compatibility
+  create-aliases:
+    needs: build-and-push-images
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    permissions:
+      contents: read
+      packages: write
+    env:
+      # Legacy name for backwards compatibility (hardcoded so it works after rename)
+      LEGACY_NAME: calibre-web-automated-book-downloader
+    steps:
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create legacy aliases
+        run: |
+          # Current image names (follows repo name)
+          STANDARD="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          LITE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-lite"
+
+          # Legacy image names (hardcoded for backwards compatibility)
+          LEGACY="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.LEGACY_NAME }}"
+          LEGACY_TOR="${LEGACY}-tor"
+          LEGACY_EXTBP="${LEGACY}-extbp"
+
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+
+          # Helper function to create alias with all standard tags
+          create_alias() {
+            local SOURCE=$1
+            local ALIAS=$2
+
+            # Always create SHA tag
+            docker buildx imagetools create -t "${ALIAS}:sha-${SHA_SHORT}" "${SOURCE}:sha-${SHA_SHORT}"
+
+            if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+              VERSION="${{ github.ref_name }}"
+              VERSION_NUM="${VERSION#v}"
+              MINOR="${VERSION_NUM%.*}"
+
+              docker buildx imagetools create -t "${ALIAS}:latest" "${SOURCE}:latest"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION_NUM}" "${SOURCE}:${VERSION_NUM}"
+              docker buildx imagetools create -t "${ALIAS}:${MINOR}" "${SOURCE}:${MINOR}"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION}" "${SOURCE}:${VERSION}"
+            else
+              docker buildx imagetools create -t "${ALIAS}:dev" "${SOURCE}:dev"
+            fi
+          }
+
+          # Create legacy aliases pointing to current images
+          # calibre-web-automated-book-downloader → standard image
+          create_alias "${STANDARD}" "${LEGACY}"
+
+          # calibre-web-automated-book-downloader-tor → standard image
+          create_alias "${STANDARD}" "${LEGACY_TOR}"
+
+          # calibre-web-automated-book-downloader-extbp → lite image
+          create_alias "${LITE}" "${LEGACY_EXTBP}"

.gitignore

@@ -227,3 +227,8 @@ pyrightconfig.json
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python
 /downloaded_files
+/.local/
+*.local.*
+AGENTS.md
+.claude/
+.playwright-mcp/

.vscode/launch.json

@@ -2,17 +2,17 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Current CWABD File",
+      "name": "Python Debugger: Current Shelfmark File",
       "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false,
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader",
-        "TEMP_DIR": "/tmp/cwa-book-downloader",
+        "INGEST_DIR": "/tmp/shelfmark",
+        "TEMP_DIR": "/tmp/shelfmark",
         "LOG_LEVEL": "DEBUG",
-        "LOG_ROOT": "/tmp/cwa-book-downloader",
+        "LOG_ROOT": "/tmp/shelfmark",
         "ENABLE_LOGGING": "true",
         "DOCKERMODE": "false",
         "DEBUG": "true",
@@ -27,7 +27,7 @@
       "preLaunchTask": "docker-compose up (dev)", // Spin up dev containers
       "postDebugTask": "docker-compose down (dev)", // Optional: tear them down
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -38,7 +38,7 @@
       "preLaunchTask": "docker-compose up (prod)",
       "postDebugTask": "docker-compose down (prod)",
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -54,9 +54,9 @@
   ],
   "compounds": [
     {
-      "name": "Launch CWA-BD",
+      "name": "Launch Shelfmark",
       "configurations": [
-        "Launch cwa-bd app.py",
+        "Launch Shelfmark app.py",
         "Launch Browser"
       ]
     }

Dockerfile

@@ -14,8 +14,9 @@ WORKDIR /frontend
 # Copy frontend package files
 COPY src/frontend/package*.json ./
 
-# Install dependencies
-RUN npm ci
+# Install dependencies (cache mount for faster rebuilds)
+RUN --mount=type=cache,target=/root/.npm \
+    npm ci
 
 # Copy frontend source
 COPY src/frontend/ ./
@@ -44,9 +45,9 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    NAME=Calibre-Web-Automated-Book-Downloader \
+    NAME=Shelfmark \
     PYTHONPATH=/app \
-    # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
+    # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
     LC_ALL=en_US.UTF-8
@@ -67,7 +68,14 @@ RUN apt-get update && \
     # For debug
     zip iputils-ping \
     # For user switching
-    sudo && \
+    sudo \
+    # --- Tor support (activated via USING_TOR=true) ---
+    tor \
+    supervisor \
+    iptables && \
+    # Configure iptables alternatives for tor.sh compatibility
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
     # Cleanup APT cache *after* all installs in this layer
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
@@ -84,12 +92,11 @@ RUN apt-get update && \
 WORKDIR /app
 
 # Install Python dependencies using pip
-# Upgrade pip first, then copy requirements and install
-# Copying requirements-base.txt separately leverages build cache
-COPY requirements-base.txt .
-RUN pip install --no-cache-dir -r requirements-base.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
+# Copying requirements files separately leverages build cache
+# Cache mount persists pip cache between builds for faster installs
+COPY requirements-base.txt requirements-shelfmark.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-base.txt
 
 # Copy application code *after* dependencies are installed
 COPY . .
@@ -100,7 +107,7 @@ COPY --from=frontend-builder /frontend/dist /app/frontend-dist
 # Final setup: permissions and directories in one layer
 # Only creating directories and setting executable bits.
 # Ownership will be handled by the entrypoint script.
-RUN mkdir -p /var/log/cwa-book-downloader /cwa-book-ingest && \
+RUN mkdir -p /var/log/shelfmark /books && \
     chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
 
 # Expose the application port
@@ -115,7 +122,7 @@ HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 
-FROM base AS cwa-bd
+FROM base AS shelfmark
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -128,44 +135,29 @@ RUN apt-get update && \
     # --- ChromeDriver ---
     chromium-driver \
     # For tkinter (pyautogui)
-    python3-tk
-
-# install additional dependencies
-COPY requirements-cwa-bd.txt .
-RUN pip install --no-cache-dir -r requirements-cwa-bd.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
-
-# Add this line to grant read/execute permissions to others
-RUN chmod -R o+rx /usr/bin/chromium && \
-    chmod -R o+rx /usr/bin/chromedriver && \
-    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
-
-# Default command to run the application entrypoint script
-CMD ["/app/entrypoint.sh"]
-
-FROM cwa-bd AS cwa-bd-tor
-
-ENV USING_TOR=true
-
-# Install Tor and dependencies
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    # --- Tor ---
-    tor \
-    # --- iptables ---
-    iptables && \
-    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
-    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
-    # Cleanup APT cache *after* all installs in this layer
+    python3-tk \
+    # For RAR extraction
+    unrar-free && \
+    # Create symlink so rarfile library can find unrar
+    ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
+    # Cleanup APT cache
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Override the default command to run Tor
+# Install additional dependencies (requirements file already copied in base stage)
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-shelfmark.txt
+
+# Grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium && \
+    chmod -R o+rx /usr/bin/chromedriver && \
+    chmod -R o+rwx /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+
+# Default command to run the application entrypoint script
 CMD ["/app/entrypoint.sh"]
 
-FROM base AS cwa-bd-extbp
+FROM base AS shelfmark-lite
 
 ENV USING_EXTERNAL_BYPASSER=true
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install dev build preview typecheck clean up down docker-build refresh
+.PHONY: help install dev build preview typecheck clean up down docker-build refresh restart
 
 # Frontend directory
 FRONTEND_DIR := src/frontend
@@ -21,6 +21,7 @@ help:
 	@echo "Backend (Docker):"
 	@echo "  up         - Start backend services"
 	@echo "  down       - Stop backend services"
+	@echo "  restart    - Restart backend services (no rebuild)"
 	@echo "  docker-build - Build Docker image"
 	@echo "  refresh    - Rebuild and restart backend services"
 
@@ -70,6 +71,11 @@ docker-build:
 	@echo "Building Docker image..."
 	docker compose -f $(COMPOSE_FILE) build
 
+# Restart backend services (no rebuild)
+restart:
+	@echo "Restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) restart
+
 # Rebuild and restart backend services
 refresh:
 	@echo "Rebuilding and restarting backend services..."

app.py

@@ -1,872 +0,0 @@
-"""Flask web application for book download service with URL rewrite support."""
-
-import io
-import logging
-import os
-import sqlite3
-import time
-from datetime import datetime, timedelta
-from functools import wraps
-from typing import Any, Dict, Tuple, Union
-
-from flask import Flask, jsonify, request, send_file, send_from_directory, session
-from flask_cors import CORS
-from flask_socketio import SocketIO, emit
-from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.security import check_password_hash
-from werkzeug.wrappers import Response
-
-import backend
-from book_manager import SearchUnavailable
-from config import BOOK_LANGUAGE, SUPPORTED_FORMATS, _SUPPORTED_BOOK_LANGUAGE
-from env import (
-    BUILD_VERSION, CALIBRE_WEB_URL, CWA_DB_PATH, DEBUG, FLASK_HOST, FLASK_PORT,
-    RELEASE_VERSION, USING_EXTERNAL_BYPASSER,
-)
-from logger import setup_logger
-from models import SearchFilters
-from websocket_manager import ws_manager
-
-logger = setup_logger(__name__)
-app = Flask(__name__)
-app.wsgi_app = ProxyFix(app.wsgi_app)  # type: ignore
-app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0  # Disable caching
-app.config['APPLICATION_ROOT'] = '/'
-
-# Socket.IO async mode.
-# We run this app under Gunicorn with a gevent websocket worker (even when DEBUG=true),
-# so Socket.IO should always use gevent here.
-async_mode = 'gevent'
-
-# Initialize Flask-SocketIO with reverse proxy support
-socketio = SocketIO(
-    app,
-    cors_allowed_origins="*",
-    async_mode=async_mode,
-    logger=False,
-    engineio_logger=False,
-    # Reverse proxy / Traefik compatibility settings
-    path='/socket.io',
-    ping_timeout=60,  # Time to wait for pong response
-    ping_interval=25,  # Send ping every 25 seconds
-    # Allow both websocket and polling for better compatibility
-    transports=['websocket', 'polling'],
-    # Enable CORS for all origins (you can restrict this in production)
-    allow_upgrades=True,
-    # Important for proxies that buffer
-    http_compression=True
-)
-
-# Initialize WebSocket manager
-ws_manager.init_app(app, socketio)
-logger.info(f"Flask-SocketIO initialized with async_mode='{async_mode}'")
-
-# Rate limiting for login attempts
-# Structure: {username: {'count': int, 'lockout_until': datetime}}
-failed_login_attempts: Dict[str, Dict[str, Any]] = {}
-MAX_LOGIN_ATTEMPTS = 10
-LOCKOUT_DURATION_MINUTES = 30
-
-def cleanup_old_lockouts() -> None:
-    """Remove expired lockout entries to prevent memory buildup."""
-    current_time = datetime.now()
-    expired_users = [
-        username for username, data in failed_login_attempts.items()
-        if 'lockout_until' in data and data['lockout_until'] < current_time
-    ]
-    for username in expired_users:
-        logger.info(f"Lockout expired for user: {username}")
-        del failed_login_attempts[username]
-
-def is_account_locked(username: str) -> bool:
-    """Check if an account is currently locked due to failed login attempts."""
-    cleanup_old_lockouts()
-    
-    if username not in failed_login_attempts:
-        return False
-    
-    lockout_until = failed_login_attempts[username].get('lockout_until')
-    if lockout_until and datetime.now() < lockout_until:
-        return True
-    
-    return False
-
-def record_failed_login(username: str, ip_address: str) -> bool:
-    """
-    Record a failed login attempt and lock account if threshold is reached.
-    Returns True if account is now locked, False otherwise.
-    """
-    if username not in failed_login_attempts:
-        failed_login_attempts[username] = {'count': 0}
-    
-    failed_login_attempts[username]['count'] += 1
-    count = failed_login_attempts[username]['count']
-    
-    logger.warning(f"Failed login attempt {count}/{MAX_LOGIN_ATTEMPTS} for user '{username}' from IP {ip_address}")
-    
-    if count >= MAX_LOGIN_ATTEMPTS:
-        lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES)
-        failed_login_attempts[username]['lockout_until'] = lockout_until
-        logger.warning(f"Account locked for user '{username}' until {lockout_until.strftime('%Y-%m-%d %H:%M:%S')} due to {count} failed login attempts")
-        return True
-    
-    return False
-
-def clear_failed_logins(username: str) -> None:
-    """Clear failed login attempts for a user after successful login."""
-    if username in failed_login_attempts:
-        del failed_login_attempts[username]
-        logger.debug(f"Cleared failed login attempts for user: {username}")
-
-# Enable CORS in development mode for local frontend development
-if DEBUG:
-    CORS(app, resources={
-        r"/*": {
-            "origins": ["http://localhost:5173", "http://127.0.0.1:5173"],
-            "supports_credentials": True,
-            "allow_headers": ["Content-Type", "Authorization"],
-            "methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"]
-        }
-    })
-
-# Custom log filter to exclude routine status endpoint polling and WebSocket noise
-class StatusEndpointFilter(logging.Filter):
-    """Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise."""
-    def filter(self, record):
-        if hasattr(record, 'getMessage'):
-            message = record.getMessage()
-            # Exclude GET /api/status requests (polling noise)
-            if 'GET /api/status' in message:
-                return False
-            # Exclude WebSocket upgrade errors (benign - falls back to polling)
-            if 'write() before start_response' in message:
-                return False
-            # Exclude the Error on request line that precedes WebSocket errors
-            if 'Error on request:' in message and record.levelno == logging.ERROR:
-                return False
-        return True
-
-
-class WebSocketErrorFilter(logging.Filter):
-    """Filter out WebSocket upgrade errors that occur in Werkzeug dev server.
-    
-    These errors are benign - Flask-SocketIO automatically falls back to polling transport.
-    The error occurs because Werkzeug's built-in server doesn't fully support WebSocket upgrades.
-    """
-    def filter(self, record):
-        # Filter out the AssertionError traceback for WebSocket upgrades
-        if record.levelno == logging.ERROR:
-            message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg)
-            # Filter out the full traceback that includes the WebSocket assertion error
-            if 'write() before start_response' in message:
-                return False
-            # Also filter the "Error on request" header that precedes it
-            if hasattr(record, 'exc_info') and record.exc_info:
-                exc_type = record.exc_info[0]
-                if exc_type and exc_type.__name__ == 'AssertionError':
-                    # Check if it's the WebSocket-related assertion
-                    exc_value = record.exc_info[1]
-                    if exc_value and 'write() before start_response' in str(exc_value):
-                        return False
-        return True
-
-# Flask logger
-app.logger.handlers = logger.handlers
-app.logger.setLevel(logger.level)
-# Also handle Werkzeug's logger
-werkzeug_logger = logging.getLogger('werkzeug')
-werkzeug_logger.handlers = logger.handlers
-werkzeug_logger.setLevel(logger.level)
-# Add filters to suppress routine status endpoint polling logs and WebSocket upgrade errors
-werkzeug_logger.addFilter(StatusEndpointFilter())
-werkzeug_logger.addFilter(WebSocketErrorFilter())
-
-# Set up authentication defaults
-# The secret key will reset every time we restart, which will
-# require users to authenticate again
-
-# Session cookie security - set to 'true' if exclusively using HTTPS
-session_cookie_secure_env = os.getenv('SESSION_COOKIE_SECURE', 'false').lower()
-SESSION_COOKIE_SECURE = session_cookie_secure_env in ['true', 'yes', '1']
-
-app.config.update(
-    SECRET_KEY = os.urandom(64),
-    SESSION_COOKIE_HTTPONLY = True,
-    SESSION_COOKIE_SAMESITE = 'Lax',
-    SESSION_COOKIE_SECURE = SESSION_COOKIE_SECURE,
-    PERMANENT_SESSION_LIFETIME = 604800  # 7 days in seconds
-)
-
-logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {session_cookie_secure_env})")
-
-def login_required(f):
-    @wraps(f)
-    def decorated_function(*args, **kwargs):
-        # If the CWA_DB_PATH variable exists, but isn't a valid
-        # path, return a server error
-        if CWA_DB_PATH is not None and not os.path.isfile(CWA_DB_PATH):
-            logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path")
-            return jsonify({"error": "Internal Server Error"}), 500
-        
-        # If no database is configured, allow access
-        if not CWA_DB_PATH:
-            return f(*args, **kwargs)
-        
-        # Check if user has a valid session
-        if 'user_id' not in session:
-            return jsonify({"error": "Unauthorized"}), 401
-        
-        return f(*args, **kwargs)
-    return decorated_function
-
-
-# Serve frontend static files
-@app.route('/assets/<path:filename>')
-def serve_frontend_assets(filename: str) -> Response:
-    """
-    Serve static assets from the built frontend.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist', 'assets'), filename)
-
-@app.route('/')
-def index() -> Response:
-    """
-    Serve the React frontend application.
-    Authentication is handled by the React app itself.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'index.html')
-
-@app.route('/logo.png')
-def logo() -> Response:
-    """
-    Serve logo from built frontend assets.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'),
-        'logo.png', mimetype='image/png')
-
-@app.route('/favicon.ico')
-@app.route('/favico<path:_>')
-def favicon(_: Any = None) -> Response:
-    """
-    Serve favicon from built frontend assets.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'),
-        'favicon.ico', mimetype='image/vnd.microsoft.icon')
-
-# Register bypasser warmup callback for when first WebSocket client connects
-# and shutdown callback for when all clients disconnect
-if not USING_EXTERNAL_BYPASSER:
-    from cloudflare_bypasser import warmup as bypasser_warmup, shutdown_if_idle as bypasser_shutdown
-    ws_manager.register_on_first_connect(bypasser_warmup)
-    ws_manager.register_on_all_disconnect(bypasser_shutdown)
-    logger.info("Registered Cloudflare bypasser warmup/shutdown on WebSocket connect/disconnect")
-
-if DEBUG:
-    import subprocess
-    if USING_EXTERNAL_BYPASSER:
-        STOP_GUI = lambda: None
-    else:
-        from cloudflare_bypasser import _reset_driver as STOP_GUI
-    @app.route('/api/debug', methods=['GET'])
-    @login_required
-    def debug() -> Union[Response, Tuple[Response, int]]:
-        """
-        This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs
-        The file will be named /tmp/cwa-book-downloader-debug.zip
-        And then return it to the user
-        """
-        try:
-            # Run the debug script
-            logger.info("Debug endpoint called, stopping GUI and generating debug info...")
-            STOP_GUI()
-            time.sleep(1)
-            result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True)
-            if result.returncode != 0:
-                raise Exception(f"Debug script failed: {result.stderr}")
-            logger.info(f"Debug script executed: {result.stdout}")
-            debug_file_path = result.stdout.strip().split('\n')[-1]
-            if not os.path.exists(debug_file_path):
-                logger.error(f"Debug zip file not found at: {debug_file_path}")
-                return jsonify({"error": "Failed to generate debug information"}), 500
-                
-            logger.info(f"Sending debug file: {debug_file_path}")
-            # Return the file to the user
-            return send_file(
-                debug_file_path,
-                mimetype='application/zip',
-                download_name=os.path.basename(debug_file_path),
-                as_attachment=True
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}")
-            return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500
-        except Exception as e:
-            logger.error_trace(f"Debug endpoint error: {e}")
-            return jsonify({"error": str(e)}), 500
-
-if DEBUG:
-    @app.route('/api/restart', methods=['GET'])
-    @login_required
-    def restart() -> Union[Response, Tuple[Response, int]]:
-        """
-        Restart the application
-        """
-        os._exit(0)
-
-@app.route('/api/search', methods=['GET'])
-@login_required
-def api_search() -> Union[Response, Tuple[Response, int]]:
-    """
-    Search for books matching the provided query.
-
-    Query Parameters:
-        query (str): Search term (ISBN, title, author, etc.)
-        isbn (str): Book ISBN
-        author (str): Book Author
-        title (str): Book Title
-        lang (str): Book Language
-        sort (str): Order to sort results
-        content (str): Content type of book
-        format (str): File format filter (pdf, epub, mobi, azw3, fb2, djvu, cbz, cbr)
-
-    Returns:
-        flask.Response: JSON array of matching books or error response.
-    """
-    query = request.args.get('query', '')
-
-    filters = SearchFilters(
-        isbn = request.args.getlist('isbn'),
-        author = request.args.getlist('author'),
-        title = request.args.getlist('title'),
-        lang = request.args.getlist('lang'),
-        sort = request.args.get('sort'),
-        content = request.args.getlist('content'),
-        format = request.args.getlist('format'),
-    )
-
-    if not query and not any(vars(filters).values()):
-        return jsonify([])
-
-    try:
-        books = backend.search_books(query, filters)
-        return jsonify(books)
-    except SearchUnavailable as e:
-        logger.warning(f"Search unavailable: {e}")
-        return jsonify({"error": str(e)}), 503
-    except Exception as e:
-        logger.error_trace(f"Search error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/info', methods=['GET'])
-@login_required
-def api_info() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get detailed book information.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON object with book details, or an error message.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        book = backend.get_book_info(book_id)
-        if book:
-            return jsonify(book)
-        return jsonify({"error": "Book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Info error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download', methods=['GET'])
-@login_required
-def api_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Queue a book for download.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON status object indicating success or failure.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        priority = int(request.args.get('priority', 0))
-        success = backend.queue_book(book_id, priority)
-        if success:
-            return jsonify({"status": "queued", "priority": priority})
-        return jsonify({"error": "Failed to queue book"}), 500
-    except Exception as e:
-        logger.error_trace(f"Download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/config', methods=['GET'])
-@login_required
-def api_config() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get application configuration for frontend.
-    """
-    try:
-        config = {
-            "calibre_web_url": CALIBRE_WEB_URL,
-            "debug": DEBUG,
-            "build_version": BUILD_VERSION,
-            "release_version": RELEASE_VERSION,
-            "book_languages": _SUPPORTED_BOOK_LANGUAGE,
-            "default_language": BOOK_LANGUAGE,
-            "supported_formats": SUPPORTED_FORMATS
-        }
-        return jsonify(config)
-    except Exception as e:
-        logger.error_trace(f"Config error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/health', methods=['GET'])
-def api_health() -> Union[Response, Tuple[Response, int]]:
-    """
-    Health check endpoint for container orchestration.
-    No authentication required.
-    
-    Returns:
-        flask.Response: JSON with status "ok".
-    """
-    return jsonify({"status": "ok"})
-
-@app.route('/api/status', methods=['GET'])
-@login_required
-def api_status() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current download queue status.
-
-    Returns:
-        flask.Response: JSON object with queue status.
-    """
-    try:
-        status = backend.queue_status()
-        return jsonify(status)
-    except Exception as e:
-        logger.error_trace(f"Status error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/localdownload', methods=['GET'])
-@login_required
-def api_local_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Download an EPUB file from local storage if available.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: The EPUB file if found, otherwise an error response.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        file_data, book_info = backend.get_book_data(book_id)
-        if file_data is None:
-            # Book data not found or not available
-            return jsonify({"error": "File not found"}), 404
-        file_name = book_info.get_filename()
-        # Prepare the file for sending to the client
-        data = io.BytesIO(file_data)
-        return send_file(
-            data,
-            download_name=file_name,
-            as_attachment=True
-        )
-
-    except Exception as e:
-        logger.error_trace(f"Local download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download/<book_id>/cancel', methods=['DELETE'])
-@login_required
-def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Cancel a download.
-
-    Path Parameters:
-        book_id (str): Book identifier to cancel
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        success = backend.cancel_download(book_id)
-        if success:
-            return jsonify({"status": "cancelled", "book_id": book_id})
-        return jsonify({"error": "Failed to cancel download or book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Cancel download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/<book_id>/priority', methods=['PUT'])
-@login_required
-def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Set priority for a queued book.
-
-    Path Parameters:
-        book_id (str): Book identifier
-
-    Request Body:
-        priority (int): New priority level (lower number = higher priority)
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'priority' not in data:
-            return jsonify({"error": "Priority not provided"}), 400
-            
-        priority = int(data['priority'])
-        success = backend.set_book_priority(book_id, priority)
-        
-        if success:
-            return jsonify({"status": "updated", "book_id": book_id, "priority": priority})
-        return jsonify({"error": "Failed to update priority or book not found"}), 404
-    except ValueError:
-        return jsonify({"error": "Invalid priority value"}), 400
-    except Exception as e:
-        logger.error_trace(f"Set priority error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/reorder', methods=['POST'])
-@login_required
-def api_reorder_queue() -> Union[Response, Tuple[Response, int]]:
-    """
-    Bulk reorder queue by setting new priorities.
-
-    Request Body:
-        book_priorities (dict): Mapping of book_id to new priority
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'book_priorities' not in data:
-            return jsonify({"error": "book_priorities not provided"}), 400
-            
-        book_priorities = data['book_priorities']
-        if not isinstance(book_priorities, dict):
-            return jsonify({"error": "book_priorities must be a dictionary"}), 400
-            
-        # Validate all priorities are integers
-        for book_id, priority in book_priorities.items():
-            if not isinstance(priority, int):
-                return jsonify({"error": f"Invalid priority for book {book_id}"}), 400
-                
-        success = backend.reorder_queue(book_priorities)
-        
-        if success:
-            return jsonify({"status": "reordered", "updated_count": len(book_priorities)})
-        return jsonify({"error": "Failed to reorder queue"}), 500
-    except Exception as e:
-        logger.error_trace(f"Reorder queue error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/order', methods=['GET'])
-@login_required
-def api_queue_order() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current queue order for display.
-
-    Returns:
-        flask.Response: JSON array of queued books with their order and priorities.
-    """
-    try:
-        queue_order = backend.get_queue_order()
-        return jsonify({"queue": queue_order})
-    except Exception as e:
-        logger.error_trace(f"Queue order error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/downloads/active', methods=['GET'])
-@login_required
-def api_active_downloads() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get list of currently active downloads.
-
-    Returns:
-        flask.Response: JSON array of active download book IDs.
-    """
-    try:
-        active_downloads = backend.get_active_downloads()
-        return jsonify({"active_downloads": active_downloads})
-    except Exception as e:
-        logger.error_trace(f"Active downloads error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/clear', methods=['DELETE'])
-@login_required
-def api_clear_completed() -> Union[Response, Tuple[Response, int]]:
-    """
-    Clear all completed, errored, or cancelled books from tracking.
-
-    Returns:
-        flask.Response: JSON with count of removed books.
-    """
-    try:
-        removed_count = backend.clear_completed()
-        
-        # Broadcast status update after clearing
-        if ws_manager:
-            ws_manager.broadcast_status_update(backend.queue_status())
-        
-        return jsonify({"status": "cleared", "removed_count": removed_count})
-    except Exception as e:
-        logger.error_trace(f"Clear completed error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.errorhandler(404)
-def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 404 (Not Found) errors.
-
-    Args:
-        error (HTTPException): The 404 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 404 status.
-    """
-    logger.warning(f"404 error: {request.url} : {error}")
-    return jsonify({"error": "Resource not found"}), 404
-
-@app.errorhandler(500)
-def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 500 (Internal Server) errors.
-
-    Args:
-        error (HTTPException): The 500 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 500 status.
-    """
-    logger.error_trace(f"500 error: {error}")
-    return jsonify({"error": "Internal server error"}), 500
-
-@app.route('/api/auth/login', methods=['POST'])
-def api_login() -> Union[Response, Tuple[Response, int]]:
-    """
-    Login endpoint that validates credentials and creates a session.
-    Includes rate limiting: 10 failed attempts = 30 minute lockout.
-    
-    Request Body:
-        username (str): Username
-        password (str): Password
-        remember_me (bool): Whether to extend session duration
-        
-    Returns:
-        flask.Response: JSON with success status or error message.
-    """
-    try:
-        # Get client IP address (handles reverse proxy forwarding)
-        ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
-        if ip_address and ',' in ip_address:
-            # X-Forwarded-For can contain multiple IPs, take the first one
-            ip_address = ip_address.split(',')[0].strip()
-        
-        data = request.get_json()
-        if not data:
-            return jsonify({"error": "No data provided"}), 400
-            
-        username = data.get('username', '').strip()
-        password = data.get('password', '')
-        remember_me = data.get('remember_me', False)
-        
-        if not username or not password:
-            return jsonify({"error": "Username and password are required"}), 400
-        
-        # Check if account is locked due to failed login attempts
-        if is_account_locked(username):
-            lockout_until = failed_login_attempts[username].get('lockout_until')
-            remaining_time = (lockout_until - datetime.now()).total_seconds() / 60
-            logger.warning(f"Login attempt blocked for locked account '{username}' from IP {ip_address}")
-            return jsonify({
-                "error": f"Account temporarily locked due to multiple failed login attempts. Try again in {int(remaining_time)} minutes."
-            }), 429
-        
-        # If the database doesn't exist, authentication always succeeds
-        if not CWA_DB_PATH:
-            session['user_id'] = username
-            session.permanent = remember_me
-            clear_failed_logins(username)
-            logger.info(f"Login successful for user '{username}' from IP {ip_address} (no DB configured)")
-            return jsonify({"success": True})
-        
-        # If the CWA_DB_PATH variable exists, but isn't a valid path, return error
-        if not os.path.isfile(CWA_DB_PATH):
-            logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path")
-            return jsonify({"error": "Database configuration error"}), 500
-        
-        # Validate credentials against database
-        try:
-            db_path = os.fspath(CWA_DB_PATH)
-            db_uri = f"file:{db_path}?mode=ro&immutable=1"
-            conn = sqlite3.connect(db_uri, uri=True)
-            cur = conn.cursor()
-            cur.execute("SELECT password FROM user WHERE name = ?", (username,))
-            row = cur.fetchone()
-            conn.close()
-            
-            # Check if user exists and password is correct
-            if not row or not row[0] or not check_password_hash(row[0], password):
-                # Record failed login attempt
-                is_now_locked = record_failed_login(username, ip_address)
-                
-                if is_now_locked:
-                    return jsonify({
-                        "error": f"Account locked due to {MAX_LOGIN_ATTEMPTS} failed login attempts. Try again in {LOCKOUT_DURATION_MINUTES} minutes."
-                    }), 429
-                else:
-                    attempts_remaining = MAX_LOGIN_ATTEMPTS - failed_login_attempts[username]['count']
-                    # Only show attempts remaining when 5 or fewer attempts remain (after 6+ failed attempts)
-                    if attempts_remaining <= 5:
-                        return jsonify({
-                            "error": f"Invalid username or password. {attempts_remaining} attempts remaining."
-                        }), 401
-                    else:
-                        return jsonify({
-                            "error": "Invalid username or password."
-                        }), 401
-            
-            # Successful authentication - create session and clear failed attempts
-            session['user_id'] = username
-            session.permanent = remember_me
-            clear_failed_logins(username)
-            logger.info(f"Login successful for user '{username}' from IP {ip_address} (remember_me={remember_me})")
-            return jsonify({"success": True})
-            
-        except Exception as e:
-            logger.error_trace(f"Database error during login: {e}")
-            return jsonify({"error": "Authentication system error"}), 500
-            
-    except Exception as e:
-        logger.error_trace(f"Login error: {e}")
-        return jsonify({"error": "Login failed"}), 500
-
-@app.route('/api/auth/logout', methods=['POST'])
-def api_logout() -> Union[Response, Tuple[Response, int]]:
-    """
-    Logout endpoint that clears the session.
-    
-    Returns:
-        flask.Response: JSON with success status.
-    """
-    try:
-        # Get client IP address (handles reverse proxy forwarding)
-        ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
-        if ip_address and ',' in ip_address:
-            ip_address = ip_address.split(',')[0].strip()
-        
-        username = session.get('user_id', 'unknown')
-        session.clear()
-        logger.info(f"Logout successful for user '{username}' from IP {ip_address}")
-        return jsonify({"success": True})
-    except Exception as e:
-        logger.error_trace(f"Logout error: {e}")
-        return jsonify({"error": "Logout failed"}), 500
-
-@app.route('/api/auth/check', methods=['GET'])
-def api_auth_check() -> Union[Response, Tuple[Response, int]]:
-    """
-    Check if user has a valid session.
-    
-    Returns:
-        flask.Response: JSON with authentication status and whether auth is required.
-    """
-    try:
-        # If no database is configured, authentication is not required
-        if not CWA_DB_PATH:
-            return jsonify({
-                "authenticated": True,
-                "auth_required": False
-            })
-        
-        # Check if user has a valid session
-        is_authenticated = 'user_id' in session
-        return jsonify({
-            "authenticated": is_authenticated,
-            "auth_required": True
-        })
-    except Exception as e:
-        logger.error_trace(f"Auth check error: {e}")
-        return jsonify({
-            "authenticated": False,
-            "auth_required": True
-        })
-
-# Catch-all route for React Router (must be last)
-# This handles client-side routing by serving index.html for any unmatched routes
-@app.route('/<path:path>')
-def catch_all(path: str) -> Response:
-    """
-    Serve the React app for any route not matched by API endpoints.
-    This allows React Router to handle client-side routing.
-    Authentication is handled by the React app itself.
-    """
-    # If the request is for an API endpoint or static file, let it 404
-    if path.startswith('api/') or path.startswith('assets/'):
-        return jsonify({"error": "Resource not found"}), 404
-    # Otherwise serve the React app
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'index.html')
-
-# WebSocket event handlers
-@socketio.on('connect')
-def handle_connect():
-    """Handle client connection."""
-    logger.info("WebSocket client connected")
-    
-    # Track the connection (triggers warmup callbacks on first connect)
-    ws_manager.client_connected()
-    
-    # Send initial status to the newly connected client
-    try:
-        status = backend.queue_status()
-        emit('status_update', status)
-    except Exception as e:
-        logger.error(f"Error sending initial status: {e}")
-
-@socketio.on('disconnect')
-def handle_disconnect():
-    """Handle client disconnection."""  
-    logger.info("WebSocket client disconnected")
-    
-    # Track the disconnection
-    ws_manager.client_disconnected()
-
-@socketio.on('request_status')
-def handle_status_request():
-    """Handle manual status request from client."""
-    try:
-        status = backend.queue_status()
-        emit('status_update', status)
-    except Exception as e:
-        logger.error(f"Error handling status request: {e}")
-        emit('error', {'message': 'Failed to get status'})
-
-logger.log_resource_usage()
-
-if __name__ == '__main__':
-    logger.info(f"Starting Flask application with WebSocket support on {FLASK_HOST}:{FLASK_PORT} (debug={DEBUG})")
-    socketio.run(
-        app,
-        host=FLASK_HOST,
-        port=FLASK_PORT,
-        debug=DEBUG,
-        allow_unsafe_werkzeug=True  # For development only
-    )

backend.py

@@ -1,513 +0,0 @@
-"""Backend logic for the book download application."""
-
-import os
-import random
-import shutil
-import subprocess
-import threading
-import time
-from concurrent.futures import Future, ThreadPoolExecutor
-from pathlib import Path
-from threading import Event, Lock
-from typing import Any, Dict, List, Optional, Tuple
-
-import book_manager
-from book_manager import SearchUnavailable
-from config import CUSTOM_SCRIPT
-from env import (
-    DOWNLOAD_PATHS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL, INGEST_DIR,
-    MAIN_LOOP_SLEEP_TIME, MAX_CONCURRENT_DOWNLOADS, TMP_DIR, USE_BOOK_TITLE,
-)
-from logger import setup_logger
-from models import BookInfo, QueueStatus, SearchFilters, book_queue
-
-logger = setup_logger(__name__)
-
-# WebSocket manager (initialized by app.py)
-try:
-    from websocket_manager import ws_manager
-except ImportError:
-    ws_manager = None
-
-# Progress update throttling - track last broadcast time per book
-_progress_last_broadcast: Dict[str, float] = {}
-_progress_lock = Lock()
-
-# Stall detection - track last activity time per download
-_last_activity: Dict[str, float] = {}
-STALL_TIMEOUT = 300  # 5 minutes without progress/status update = stalled
-
-def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
-    """Search for books matching the query.
-    
-    Args:
-        query: Search term
-        filters: Search filters object
-        
-    Returns:
-        List[Dict]: List of book information dictionaries
-    """
-    try:
-        books = book_manager.search_books(query, filters)
-        return [_book_info_to_dict(book) for book in books]
-    except SearchUnavailable as e:
-        logger.warning(f"Search unavailable: {e}")
-        raise
-    except Exception as e:
-        logger.error_trace(f"Error searching books: {e}")
-        return []
-
-def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
-    """Get detailed information for a specific book.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Optional[Dict]: Book information dictionary if found
-    """
-    try:
-        book = book_manager.get_book_info(book_id)
-        return _book_info_to_dict(book)
-    except Exception as e:
-        logger.error_trace(f"Error getting book info: {e}")
-        return None
-
-def queue_book(book_id: str, priority: int = 0) -> bool:
-    """Add a book to the download queue with specified priority.
-    
-    Args:
-        book_id: Book identifier
-        priority: Priority level (lower number = higher priority)
-        
-    Returns:
-        bool: True if book was successfully queued
-    """
-    try:
-        book_info = book_manager.get_book_info(book_id)
-        book_queue.add(book_id, book_info, priority)
-        logger.info(f"Book queued with priority {priority}: {book_info.title}")
-        
-        # Broadcast status update via WebSocket
-        if ws_manager:
-            ws_manager.broadcast_status_update(queue_status())
-        
-        return True
-    except Exception as e:
-        logger.error_trace(f"Error queueing book: {e}")
-        return False
-
-def queue_status() -> Dict[str, Dict[str, Any]]:
-    """Get current status of the download queue.
-    
-    Returns:
-        Dict: Queue status organized by status type with serialized book data
-    """
-    status = book_queue.get_status()
-    for _, books in status.items():
-        for _, book_info in books.items():
-            if book_info.download_path:
-                if not os.path.exists(book_info.download_path):
-                    book_info.download_path = None
-
-    # Convert Enum keys to strings and BookInfo objects to dicts for JSON serialization
-    return {
-        status_type.value: {
-            book_id: _book_info_to_dict(book_info)
-            for book_id, book_info in books.items()
-        }
-        for status_type, books in status.items()
-    }
-
-def get_book_data(book_id: str) -> Tuple[Optional[bytes], BookInfo]:
-    """Get book data for a specific book, including its title.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Tuple[Optional[bytes], str]: Book data if available, and the book title
-    """
-    try:
-        book_info = book_queue._book_data[book_id]
-        path = book_info.download_path
-        with open(path, "rb") as f:
-            return f.read(), book_info
-    except Exception as e:
-        logger.error_trace(f"Error getting book data: {e}")
-        if book_info:
-            book_info.download_path = None
-        return None, book_info if book_info else BookInfo(id=book_id, title="Unknown")
-
-def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
-    """Convert BookInfo object to dictionary representation."""
-    return {
-        key: value for key, value in book.__dict__.items()
-        if value is not None
-    }
-
-def _prepare_download_folder(book_info: BookInfo) -> Path:
-    """Prepare final content-type subdir"""
-    content = book_info.content
-    content_dir = DOWNLOAD_PATHS.get(content) if content and content in DOWNLOAD_PATHS else INGEST_DIR
-    os.makedirs(content_dir, exist_ok=True)
-    return content_dir
-
-def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Optional[str]:
-    """Download and process a book with cancellation support.
-    
-    Args:
-        book_id: Book identifier
-        cancel_flag: Threading event to signal cancellation
-        
-    Returns:
-        str: Path to the downloaded book if successful, None otherwise
-    """
-    try:
-        # Check for cancellation before starting
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before starting: {book_id}")
-            return None
-            
-        book_info = book_queue._book_data[book_id]
-        logger.info(f"Starting download: {book_info.title}")
-
-        if not book_info.download_urls:
-            raise ValueError(f"No download URLs available for {book_id}")
-
-        # get_filename() resolves format as side effect
-        full_name = book_info.get_filename()
-        book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}"
-        book_path = TMP_DIR / book_name
-
-        # Check cancellation before download
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before book manager call: {book_id}")
-            return None
-        
-        progress_callback = lambda progress: update_download_progress(book_id, progress)
-        status_callback = lambda status, message=None: update_download_status(book_id, status, message)
-        
-        # Set status to resolving immediately when processing starts
-        update_download_status(book_id, "resolving")
-        
-        success_download_url = book_manager.download_book(book_info, book_path, progress_callback, cancel_flag, status_callback)
-        
-        # Stop progress updates
-        cancel_flag.wait(0.1)  # Brief pause for progress thread cleanup
-        
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during download: {book_id}")
-            # Clean up partial download
-            if book_path.exists():
-                book_path.unlink()
-            return None
-            
-        if not success_download_url:
-            raise Exception("Unknown error downloading book")
-
-        # Check cancellation before post-processing
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before post-processing: {book_id}")
-            if book_path.exists():
-                book_path.unlink()
-            return None
-
-        logger.debug(f"Post-processing download: {book_info.title}")
-
-        if CUSTOM_SCRIPT:
-            logger.info(f"Running custom script: {CUSTOM_SCRIPT}")
-            subprocess.run([CUSTOM_SCRIPT, book_path])
-        
-        # Regenerate filename with fallback to successful download URL for format
-        full_name = book_info.get_filename(success_download_url)
-        book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}"
-
-        final_dir = _prepare_download_folder(book_info)
-        intermediate_path = final_dir / f"{book_id}.crdownload"
-        final_path = final_dir / book_name
-        
-        # Handle file already exists - add suffix to avoid overwrite
-        if final_path.exists():
-            base = final_path.stem
-            ext = final_path.suffix
-            counter = 1
-            while final_path.exists():
-                final_path = final_dir / f"{base}_{counter}{ext}"
-                counter += 1
-            logger.info(f"File already exists, saving as: {final_path.name}")
-        
-        if os.path.exists(book_path):
-            logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}")
-            try:
-                shutil.move(book_path, intermediate_path)
-            except Exception as e:
-                try:
-                    logger.debug(f"Error moving book: {e}, will try copying instead")
-                    shutil.move(book_path, intermediate_path)
-                except Exception as e:
-                    logger.debug(f"Error copying book: {e}, will try copying without permissions instead")
-                    shutil.copyfile(book_path, intermediate_path)
-                os.remove(book_path)
-            
-            # Final cancellation check before completing
-            if cancel_flag.is_set():
-                logger.info(f"Download cancelled before final rename: {book_id}")
-                if intermediate_path.exists():
-                    intermediate_path.unlink()
-                return None
-                
-            os.rename(intermediate_path, final_path)
-            logger.info(f"Download completed successfully: {book_info.title}")
-            
-        return str(final_path)
-    except Exception as e:
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during error handling: {book_id}")
-        else:
-            logger.error_trace(f"Error downloading book: {e}")
-        return None
-
-def update_download_progress(book_id: str, progress: float) -> None:
-    """Update download progress with throttled WebSocket broadcasts.
-
-    Progress is always stored in the queue, but WebSocket broadcasts are
-    throttled to avoid flooding clients with updates. Broadcasts occur:
-    - At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds
-    - Always at 0% (start) and 100% (complete)
-    - On significant progress jumps (>10%)
-    """
-    book_queue.update_progress(book_id, progress)
-
-    # Track activity for stall detection
-    with _progress_lock:
-        _last_activity[book_id] = time.time()
-    
-    # Broadcast progress via WebSocket with throttling
-    if ws_manager:
-        current_time = time.time()
-        should_broadcast = False
-        
-        with _progress_lock:
-            last_broadcast = _progress_last_broadcast.get(book_id, 0)
-            last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0)
-            time_elapsed = current_time - last_broadcast
-            
-            # Always broadcast at start (0%) or completion (>=99%)
-            if progress <= 1 or progress >= 99:
-                should_broadcast = True
-            # Broadcast if enough time has passed (convert interval from seconds)
-            elif time_elapsed >= DOWNLOAD_PROGRESS_UPDATE_INTERVAL:
-                should_broadcast = True
-            # Broadcast on significant progress jumps (>10%)
-            elif progress - last_progress >= 10:
-                should_broadcast = True
-            
-            if should_broadcast:
-                _progress_last_broadcast[book_id] = current_time
-                _progress_last_broadcast[f"{book_id}_progress"] = progress
-        
-        if should_broadcast:
-            ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
-
-def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
-    """Update download status with optional detailed message.
-    
-    Args:
-        book_id: Book identifier
-        status: Status string (e.g., 'resolving', 'downloading')
-        message: Optional detailed status message for UI display
-    """
-    # Map string status to QueueStatus enum
-    status_map = {
-        'queued': QueueStatus.QUEUED,
-        'resolving': QueueStatus.RESOLVING,
-        'downloading': QueueStatus.DOWNLOADING,
-        'complete': QueueStatus.COMPLETE,
-        'available': QueueStatus.AVAILABLE,
-        'error': QueueStatus.ERROR,
-        'done': QueueStatus.DONE,
-        'cancelled': QueueStatus.CANCELLED,
-    }
-    
-    queue_status_enum = status_map.get(status.lower())
-    if queue_status_enum:
-        book_queue.update_status(book_id, queue_status_enum)
-
-        # Track activity for stall detection
-        with _progress_lock:
-            _last_activity[book_id] = time.time()
-
-        # Update status message if provided (empty string clears the message)
-        if message is not None:
-            book_queue.update_status_message(book_id, message)
-
-        # Broadcast status update via WebSocket
-        if ws_manager:
-            ws_manager.broadcast_status_update(queue_status())
-
-def cancel_download(book_id: str) -> bool:
-    """Cancel a download.
-    
-    Args:
-        book_id: Book identifier to cancel
-        
-    Returns:
-        bool: True if cancellation was successful
-    """
-    result = book_queue.cancel_download(book_id)
-    
-    # Broadcast status update via WebSocket
-    if result and ws_manager and ws_manager.is_enabled():
-        ws_manager.broadcast_status_update(queue_status())
-    
-    return result
-
-def set_book_priority(book_id: str, priority: int) -> bool:
-    """Set priority for a queued book.
-    
-    Args:
-        book_id: Book identifier
-        priority: New priority level (lower = higher priority)
-        
-    Returns:
-        bool: True if priority was successfully changed
-    """
-    return book_queue.set_priority(book_id, priority)
-
-def reorder_queue(book_priorities: Dict[str, int]) -> bool:
-    """Bulk reorder queue.
-    
-    Args:
-        book_priorities: Dict mapping book_id to new priority
-        
-    Returns:
-        bool: True if reordering was successful
-    """
-    return book_queue.reorder_queue(book_priorities)
-
-def get_queue_order() -> List[Dict[str, any]]:
-    """Get current queue order for display."""
-    return book_queue.get_queue_order()
-
-def get_active_downloads() -> List[str]:
-    """Get list of currently active downloads."""
-    return book_queue.get_active_downloads()
-
-def clear_completed() -> int:
-    """Clear all completed downloads from tracking."""
-    return book_queue.clear_completed()
-
-def _cleanup_progress_tracking(book_id: str) -> None:
-    """Clean up progress tracking data for a completed/cancelled download."""
-    with _progress_lock:
-        _progress_last_broadcast.pop(book_id, None)
-        _progress_last_broadcast.pop(f"{book_id}_progress", None)
-        _last_activity.pop(book_id, None)
-
-def _process_single_download(book_id: str, cancel_flag: Event) -> None:
-    """Process a single download job."""
-    try:
-        # Status will be updated through callbacks during download process
-        # (resolving -> downloading -> complete)
-        download_path = _download_book_with_cancellation(book_id, cancel_flag)
-        
-        # Clean up progress tracking
-        _cleanup_progress_tracking(book_id)
-        
-        if cancel_flag.is_set():
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-            # Broadcast cancellation
-            if ws_manager:
-                ws_manager.broadcast_status_update(queue_status())
-            return
-            
-        if download_path:
-            book_queue.update_download_path(book_id, download_path)
-            new_status = QueueStatus.COMPLETE
-        else:
-            new_status = QueueStatus.ERROR
-            
-        book_queue.update_status(book_id, new_status)
-        
-        # Broadcast final status (completed or error)
-        if ws_manager:
-            ws_manager.broadcast_status_update(queue_status())
-        
-        
-    except Exception as e:
-        # Clean up progress tracking even on error
-        _cleanup_progress_tracking(book_id)
-        
-        if not cancel_flag.is_set():
-            logger.error_trace(f"Error in download processing: {e}")
-            book_queue.update_status(book_id, QueueStatus.ERROR)
-            # Set error message if not already set by download_book()
-            if book_id in book_queue._book_data and not book_queue._book_data[book_id].status_message:
-                book_queue.update_status_message(book_id, f"Download failed: {type(e).__name__}: {str(e)}")
-        else:
-            logger.info(f"Download cancelled: {book_id}")
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-        
-        # Broadcast error/cancelled status
-        if ws_manager:
-            ws_manager.broadcast_status_update(queue_status())
-
-def concurrent_download_loop() -> None:
-    """Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
-    logger.info(f"Starting concurrent download loop with {MAX_CONCURRENT_DOWNLOADS} workers")
-    
-    with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_DOWNLOADS, thread_name_prefix="BookDownload") as executor:
-        active_futures: Dict[Future, str] = {}  # Track active download futures
-        
-        while True:
-            # Clean up completed futures
-            completed_futures = [f for f in active_futures if f.done()]
-            for future in completed_futures:
-                book_id = active_futures.pop(future)
-                try:
-                    future.result()  # This will raise any exceptions from the worker
-                except Exception as e:
-                    logger.error_trace(f"Future exception for {book_id}: {e}")
-
-            # Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
-            current_time = time.time()
-            with _progress_lock:
-                for future, book_id in list(active_futures.items()):
-                    last_active = _last_activity.get(book_id, current_time)
-                    if current_time - last_active > STALL_TIMEOUT:
-                        logger.warning(f"Download stalled for {book_id}, cancelling")
-                        book_queue.cancel_download(book_id)
-                        book_queue.update_status_message(book_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)")
-
-            # Start new downloads if we have capacity
-            while len(active_futures) < MAX_CONCURRENT_DOWNLOADS:
-                next_download = book_queue.get_next()
-                if not next_download:
-                    break
-
-                # Stagger concurrent downloads to avoid rate limiting on shared download servers
-                # Only delay if other downloads are already active
-                if active_futures:
-                    stagger_delay = random.uniform(2, 5)
-                    logger.debug(f"Staggering download start by {stagger_delay:.1f}s")
-                    time.sleep(stagger_delay)
-
-                book_id, cancel_flag = next_download
-
-                # Submit download job to thread pool
-                future = executor.submit(_process_single_download, book_id, cancel_flag)
-                active_futures[future] = book_id
-            
-            # Brief sleep to prevent busy waiting
-            time.sleep(MAIN_LOOP_SLEEP_TIME)
-
-# Start concurrent download coordinator
-download_coordinator_thread = threading.Thread(
-    target=concurrent_download_loop,
-    daemon=True,
-    name="DownloadCoordinator"
-)
-download_coordinator_thread.start()
-
-logger.info(f"Download system initialized with {MAX_CONCURRENT_DOWNLOADS} concurrent workers")

book_manager.py

@@ -1,785 +0,0 @@
-"""Book download manager handling search and retrieval operations."""
-
-import itertools
-import json
-import re
-import time
-from pathlib import Path
-from threading import Event
-from typing import Callable, Dict, List, Optional
-from urllib.parse import quote
-
-from bs4 import BeautifulSoup, NavigableString, Tag
-
-import downloader
-import network
-from config import BOOK_LANGUAGE, SUPPORTED_FORMATS
-from env import AA_DONATOR_KEY, ALLOW_USE_WELIB, DEBUG_SKIP_SOURCES, DOWNLOAD_PATHS, PRIORITIZE_WELIB, USE_CF_BYPASS
-from logger import setup_logger
-from models import BookInfo, SearchFilters
-
-logger = setup_logger(__name__)
-
-# Round-robin counter for AA slow download source rotation
-# Distributes concurrent downloads across different partner mirrors
-_aa_slow_rotation = itertools.count()
-
-if DEBUG_SKIP_SOURCES:
-    logger.warning("DEBUG_SKIP_SOURCES active: skipping sources %s", DEBUG_SKIP_SOURCES)
-
-
-class SearchUnavailable(Exception):
-    """Raised when Anna's Archive cannot be reached via any mirror/DNS."""
-    pass
-
-
-
-def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
-    """Search for books matching the query.
-
-    Args:
-        query: Search term (ISBN, title, author, etc.)
-
-    Returns:
-        List[BookInfo]: List of matching books
-
-    Raises:
-        Exception: If no books found or parsing fails
-    """
-    query_html = quote(query)
-
-    if filters.isbn:
-        # ISBNs are included in query string
-        isbns = " || ".join(
-            [f"('isbn13:{isbn}' || 'isbn10:{isbn}')" for isbn in filters.isbn]
-        )
-        query_html = quote(f"({isbns}) {query}")
-
-    filters_query = ""
-
-    for value in filters.lang or BOOK_LANGUAGE:
-        if value != "all":
-            filters_query += f"&lang={quote(value)}"
-
-    if filters.sort:
-        filters_query += f"&sort={quote(filters.sort)}"
-
-    if filters.content:
-        for value in filters.content:
-            filters_query += f"&content={quote(value)}"
-
-    # Handle format filter
-    formats_to_use = filters.format if filters.format else SUPPORTED_FORMATS
-
-    index = 1
-    for filter_type, filter_values in vars(filters).items():
-        if filter_type == "author" or filter_type == "title" and filter_values:
-            for value in filter_values:
-                filters_query += (
-                    f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}"
-                )
-                index += 1
-
-    selector = network.AAMirrorSelector()
-
-    url = (
-        f"{network.get_aa_base_url()}"
-        f"/search?index=&page=1&display=table"
-        f"&acc=aa_download&acc=external_download"
-        f"&ext={'&ext='.join(formats_to_use)}"
-        f"&q={query_html}"
-        f"{filters_query}"
-    )
-
-    html = downloader.html_get_page(url, selector=selector)
-    if not html:
-        # Network/mirror exhaustion path bubbles up so API can notify clients
-        raise SearchUnavailable("Unable to reach Anna's Archive. Network restricted or mirrors are blocked.")
-
-    if "No files found." in html:
-        logger.info(f"No books found for query: {query}")
-        return []
-
-    soup = BeautifulSoup(html, "html.parser")
-    tbody: Tag | NavigableString | None = soup.find("table")
-
-    if not tbody:
-        logger.warning(f"No results table found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    books = []
-    if isinstance(tbody, Tag):
-        for line_tr in tbody.find_all("tr"):
-            try:
-                book = _parse_search_result_row(line_tr)
-                if book:
-                    books.append(book)
-            except Exception as e:
-                logger.error_trace(f"Failed to parse search result row: {e}")
-
-    books.sort(
-        key=lambda x: (
-            SUPPORTED_FORMATS.index(x.format)
-            if x.format in SUPPORTED_FORMATS
-            else len(SUPPORTED_FORMATS)
-        )
-    )
-
-    return books
-
-
-def _parse_search_result_row(row: Tag) -> Optional[BookInfo]:
-    """Parse a single search result row into a BookInfo object."""
-    try:
-        # Skip ad rows
-        if row.text.strip().lower().startswith("your ad here"):
-            return None
-        cells = row.find_all("td")
-        preview_img = cells[0].find("img")
-        preview = preview_img["src"] if preview_img else None
-
-        return BookInfo(
-            id=row.find_all("a")[0]["href"].split("/")[-1],
-            preview=preview,
-            title=cells[1].find("span").next,
-            author=cells[2].find("span").next,
-            publisher=cells[3].find("span").next,
-            year=cells[4].find("span").next,
-            language=cells[7].find("span").next,
-            content=cells[8].find("span").next.lower(),
-            format=cells[9].find("span").next.lower(),
-            size=cells[10].find("span").next,
-        )
-    except Exception as e:
-        logger.error_trace(f"Error parsing search result row: {e}")
-        return None
-
-
-def get_book_info(book_id: str) -> BookInfo:
-    """Get detailed information for a specific book.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-
-    Returns:
-        BookInfo: Detailed book information
-    """
-    url = f"{network.get_aa_base_url()}/md5/{book_id}"
-    selector = network.AAMirrorSelector()
-    html = downloader.html_get_page(url, selector=selector)
-
-    if not html:
-        raise Exception(f"Failed to fetch book info for ID: {book_id}")
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    return _parse_book_info_page(soup, book_id)
-
-
-def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
-    """Parse the book info page HTML into a BookInfo object."""
-    data = soup.select_one("body > main > div:nth-of-type(1)")
-
-    if not data:
-        raise Exception(f"Failed to parse book info for ID: {book_id}")
-
-    preview: str = ""
-
-    node = data.select_one("div:nth-of-type(1) > img")
-    if node:
-        preview_value = node.get("src", "")
-        if isinstance(preview_value, list):
-            preview = preview_value[0]
-        else:
-            preview = preview_value
-
-    data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
-    divs = list(data.children)
-
-    # Collect download URLs by source type (lists preserve page order, dedup inline)
-    slow_urls_no_waitlist: list[str] = []
-    slow_urls_with_waitlist: list[str] = []
-    external_urls_libgen: list[str] = []
-    external_urls_z_lib: list[str] = []
-
-    def _append_unique(lst: list[str], href: str) -> None:
-        if href and href not in lst:
-            lst.append(href)
-
-    for anchor in soup.find_all("a"):
-        try:
-            text = anchor.text.strip().lower()
-            href = anchor.get("href", "")
-            next_text = ""
-            if anchor.next and anchor.next.next:
-                next_text = getattr(anchor.next.next, 'text', str(anchor.next.next)).strip().lower()
-
-            if text.startswith("slow partner server") and "waitlist" in next_text:
-                if "no waitlist" in next_text:
-                    _append_unique(slow_urls_no_waitlist, href)
-                else:
-                    _append_unique(slow_urls_with_waitlist, href)
-            elif 'libgen.li' in href:
-                # Normalize libgen domains
-                libgen_url = re.sub(r'libgen\.(li|lc|is|bz|st)', 'libgen.gl', href)
-                _append_unique(external_urls_libgen, libgen_url)
-            elif text.startswith("z-lib") and ".onion/" not in href:
-                _append_unique(external_urls_z_lib, href)
-        except:
-            pass
-
-    logger.debug(
-        "Source inventory for %s -> aa_no_wait=%d, aa_wait=%d, libgen=%d, zlib=%d",
-        book_id,
-        len(slow_urls_no_waitlist),
-        len(slow_urls_with_waitlist),
-        len(external_urls_libgen),
-        len(external_urls_z_lib),
-    )
-
-    urls = []
-
-    # Priority: reliable sources first, then external fallbacks
-    # 1. AA slow (no waitlist) - instant but can be slow
-    # 2. Libgen - instant, external
-    # 3. AA slow (waitlist) - has countdown timer but faster once started
-    # Note: Z-Library disabled - download tokens are session-bound
-    urls += slow_urls_no_waitlist if USE_CF_BYPASS else []
-    urls += external_urls_libgen
-    urls += slow_urls_with_waitlist if USE_CF_BYPASS else []
-
-    for i in range(len(urls)):
-        urls[i] = downloader.get_absolute_url(network.get_aa_base_url(), urls[i])
-
-    # Remove empty urls
-    urls = [url for url in urls if url != ""]
-
-    # Tag AA slow URLs with detailed source type for skip/retry tracking
-    base_url = network.get_aa_base_url()
-    for rel_url in slow_urls_no_waitlist:
-        abs_url = downloader.get_absolute_url(base_url, rel_url)
-        if abs_url:
-            _url_source_types[abs_url] = "aa-slow-nowait"
-    for rel_url in slow_urls_with_waitlist:
-        abs_url = downloader.get_absolute_url(base_url, rel_url)
-        if abs_url:
-            _url_source_types[abs_url] = "aa-slow-wait"
-
-    # Filter out divs that are not text
-    original_divs = divs
-    divs = [div for div in divs if div.text.strip() != ""]
-
-    all_details = _find_in_divs(divs, " · ")
-    format = ""
-    size = ""
-    content = ""
-    
-    for _details in all_details:
-        _details = _details.split(" · ")
-        for f in _details:
-            if format == "" and f.strip().lower() in SUPPORTED_FORMATS:
-                format = f.strip().lower()
-            if size == "" and any(u in f.strip().lower() for u in ["mb", "kb", "gb"]):
-                # Preserve original case but uppercase the unit (e.g., "5.2 mb" -> "5.2 MB")
-                size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE)
-            if content == "":
-                for ct in DOWNLOAD_PATHS.keys():
-                    if ct in f.strip().lower():
-                        content = ct
-                        break
-        if format == "" or size == "":
-            for f in _details:
-                stripped = f.strip().lower()
-                if format == "" and stripped and " " not in stripped:
-                    format = stripped
-                if size == "" and "." in stripped:
-                    # Uppercase any size units
-                    size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE)
-    
-    book_title = _find_in_divs(divs, "🔍")[0].strip("🔍").strip()
-
-    # Extract basic information
-    description = _extract_book_description(soup)
-
-    book_info = BookInfo(
-        id=book_id,
-        preview=preview,
-        title=book_title,
-        content=content,
-        publisher=_find_in_divs(divs, "icon-[mdi--company]", is_class=True)[0],
-        author=_find_in_divs(divs, "icon-[mdi--user-edit]", is_class=True)[0],
-        format=format,
-        size=size,
-        description=description,
-        download_urls=urls,
-    )
-
-    # Extract additional metadata
-    info = _extract_book_metadata(original_divs[-6])
-    book_info.info = info
-
-    # Set language and year from metadata if available
-    if info.get("Language"):
-        book_info.language = info["Language"][0]
-    if info.get("Year"):
-        book_info.year = info["Year"][0]
-
-    # TODO :
-    # Backfill missing metadata from original book
-    # To do this, we need to cache the results of search_books() in some kind of LRU
-
-    return book_info
-
-def _find_in_divs(divs: List, text: str, is_class: bool = False) -> List[str]:
-    """Find divs containing text or having a specific class."""
-    results = []
-    for div in divs:
-        if is_class:
-            if div.find(class_=text):
-                results.append(div.text.strip())
-        elif text in div.text.strip():
-            results.append(div.text.strip())
-    return results
-
-# Download source definitions: (log_label, friendly_name, url_patterns)
-_DOWNLOAD_SOURCES = [
-    ("welib", "Welib", ["welib.org"]),
-    ("aa-fast", "Anna's Archive (Fast)", ["/dyn/api/fast_download"]),
-    ("aa-slow-wait", "Anna's Archive (Waitlist)", []),  # Matched via _url_source_types
-    ("aa-slow-nowait", "Anna's Archive", []),  # Matched via _url_source_types
-    ("aa-slow", "Anna's Archive", ["/slow_download/", "annas-"]),  # Fallback for untagged AA URLs
-    ("libgen", "Libgen", ["libgen"]),
-    ("zlib", "Z-Library", ["z-lib", "zlibrary"]),
-]
-
-# Track detailed source types for AA slow URLs (populated during get_book_info)
-_url_source_types: dict[str, str] = {}
-
-
-def _get_source_info(link: str) -> tuple[str, str]:
-    """Get source label and friendly name for a download link.
-
-    Args:
-        link: Download URL
-
-    Returns:
-        Tuple of (log_label, friendly_name)
-    """
-    # Check detailed source type mapping first (for AA slow distinction)
-    if link in _url_source_types:
-        detailed_label = _url_source_types[link]
-        for log_label, friendly_name, _ in _DOWNLOAD_SOURCES:
-            if log_label == detailed_label:
-                return log_label, friendly_name
-
-    for log_label, friendly_name, patterns in _DOWNLOAD_SOURCES:
-        if patterns and any(pattern in link for pattern in patterns):
-            return log_label, friendly_name
-    return "unknown", "Mirror"
-
-
-def _label_source(link: str) -> str:
-    """Get lightweight source tag for logging/metrics."""
-    return _get_source_info(link)[0]
-
-
-def _friendly_source_name(link: str) -> str:
-    """Get user-friendly name for a download source."""
-    return _get_source_info(link)[1]
-
-def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> list[str]:
-    """Get download URLs from welib.org (bypasser required)."""
-    if not ALLOW_USE_WELIB:
-        return []
-    url = f"https://welib.org/md5/{book_id}"
-    logger.info(f"Fetching welib.org download URLs for {book_id}")
-    try:
-        html = downloader.html_get_page(url, use_bypasser=True, selector=selector or network.AAMirrorSelector(), cancel_flag=cancel_flag)
-    except Exception as exc:
-        logger.error_trace(f"Welib fetch failed for {book_id}: {exc}")
-        return []
-    if not html:
-        logger.warning(f"Welib page empty for {book_id}")
-        return []
-    
-    soup = BeautifulSoup(html, "html.parser")
-    links = [
-        downloader.get_absolute_url(url, a["href"])
-        for a in soup.find_all("a", href=True)
-        if "/slow_download/" in a["href"]
-    ]
-    return list(dict.fromkeys(links))  # Dedupe while preserving order
-
-def _get_next_value_div(label_div: Tag) -> Optional[Tag]:
-    """Find the next sibling div that holds the value for a metadata label."""
-    sibling = label_div.next_sibling
-    while sibling:
-        if isinstance(sibling, Tag) and sibling.name == "div":
-            return sibling
-        sibling = sibling.next_sibling
-    return None
-
-def _extract_book_description(soup: BeautifulSoup) -> Optional[str]:
-    """Extract the primary or alternative description from the book page."""
-    container = soup.select_one(".js-md5-top-box-description")
-    if not container:
-        return None
-
-    description: Optional[str] = None
-    alternative: Optional[str] = None
-
-    label_divs = container.select("div.text-xs.text-gray-500.uppercase")
-    for label_div in label_divs:
-        label_text = label_div.get_text(strip=True).lower()
-        value_div = _get_next_value_div(label_div)
-        if not value_div:
-            continue
-
-        value_text = value_div.get_text(separator=" ", strip=True)
-        if not value_text:
-            continue
-
-        if label_text == "description":
-            return value_text
-        if label_text == "alternative description" and not alternative:
-            alternative = value_text
-
-    if alternative:
-        return alternative
-
-    # Fallback to the first text block inside the description container
-    fallback_div = container.find("div", class_="mb-1")
-    if fallback_div:
-        fallback_text = fallback_div.get_text(separator=" ", strip=True)
-        if fallback_text:
-            return fallback_text
-
-    return None
-
-def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]:
-    """Extract metadata from book info divs."""
-    info: Dict[str, List[str]] = {}
-
-    # Process the first set of metadata
-    sub_datas = metadata_divs.find_all("div")[0]
-    sub_datas = list(sub_datas.children)
-    for sub_data in sub_datas:
-        if sub_data.text.strip() == "":
-            continue
-        sub_data = list(sub_data.children)
-        key = sub_data[0].text.strip()
-        value = sub_data[1].text.strip()
-        if key not in info:
-            info[key] = set()
-        info[key].add(value)
-    
-    # make set into list
-    for key, value in info.items():
-        info[key] = list(value)
-
-    # Filter relevant metadata
-    relevant_prefixes = [
-        "ISBN-",
-        "ALTERNATIVE",
-        "ASIN",
-        "Goodreads",
-        "Language",
-        "Year",
-    ]
-    return {
-        k.strip(): v
-        for k, v in info.items()
-        if any(k.lower().startswith(prefix.lower()) for prefix in relevant_prefixes)
-        and "filename" not in k.lower()
-    }
-
-
-# After N consecutive failures of the same source type, skip remaining sources of that type
-SOURCE_FAILURE_THRESHOLD = 4
-
-# Minimum valid file size in bytes (10KB) - anything smaller is likely an error page
-MIN_VALID_FILE_SIZE = 10 * 1024
-
-
-def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None) -> Optional[str]:
-    """Download a book from available sources.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-        title: Book title for logging
-        progress_callback: Optional callback for download progress updates
-        cancel_flag: Optional cancellation flag
-        status_callback: Optional callback for status updates (status, message)
-
-    Returns:
-        str: Download URL if successful, None otherwise
-    """
-
-    selector = network.AAMirrorSelector()
-
-    if len(book_info.download_urls) == 0:
-        book_info = get_book_info(book_info.id)
-    download_links = list(book_info.download_urls)
-
-    # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources.
-    if AA_DONATOR_KEY != "":
-        download_links.insert(
-            0,
-            f"{network.get_aa_base_url()}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}",
-        )
-
-    # Preserve order but drop duplicates to avoid retrying the same host
-    download_links = list(dict.fromkeys(download_links))
-
-    # Round-robin rotation for AA slow download URLs to distribute load across mirrors
-    # This prevents all concurrent downloads from hitting the same partner server first
-    # Rotate aa-slow-nowait and aa-slow-wait independently to preserve priority ordering
-    rotation_value = next(_aa_slow_rotation)
-
-    def _rotate_category_in_place(links: list, source_type: str) -> int:
-        """Rotate URLs of a specific source type within the list, preserving their positions."""
-        indices = [i for i, u in enumerate(links) if _url_source_types.get(u) == source_type]
-        if len(indices) <= 1:
-            return 0
-        rotation = rotation_value % len(indices)
-        if rotation == 0:
-            return 0
-        # Extract values, rotate, put back
-        values = [links[i] for i in indices]
-        rotated = values[rotation:] + values[:rotation]
-        for idx, val in zip(indices, rotated):
-            links[idx] = val
-        return rotation
-
-    nowait_rotation = _rotate_category_in_place(download_links, "aa-slow-nowait")
-    wait_rotation = _rotate_category_in_place(download_links, "aa-slow-wait")
-
-    if nowait_rotation or wait_rotation:
-        logger.info(f"AA source rotation: nowait={nowait_rotation}, wait={wait_rotation}")
-
-    links_queue = download_links
-    
-    # Fetch welib URLs upfront when prioritized
-    welib_fallback_loaded = "welib" in DEBUG_SKIP_SOURCES  # Skip welib entirely if in debug skip list
-    if USE_CF_BYPASS and PRIORITIZE_WELIB and ALLOW_USE_WELIB and not welib_fallback_loaded:
-        logger.info("Fetching welib.org download URLs (PRIORITIZE_WELIB enabled)")
-        if status_callback:
-            status_callback("resolving", "Fetching welib sources...")
-        welib_links = _get_download_urls_from_welib(book_info.id, selector=selector, cancel_flag=cancel_flag)
-        if welib_links:
-            links_queue = welib_links + [l for l in links_queue if l not in welib_links]
-        welib_fallback_loaded = True
-    
-    total_sources = len(links_queue)
-    
-    # Handle case where no download sources are available
-    if total_sources == 0:
-        logger.warning(f"No download sources available for: {book_info.title}")
-        if status_callback:
-            status_callback("error", "No download sources found")
-        return None
-    
-    # Track consecutive failures per source type to skip after threshold
-    source_failures: dict[str, int] = {}
-    # Iterate with index so we can append welib links later
-    idx = 0
-    while idx < len(links_queue):
-        link = links_queue[idx]
-        source_label = _label_source(link)
-        friendly_name = _friendly_source_name(link)
-
-        # Debug: skip sources for testing fallback chains
-        if source_label in DEBUG_SKIP_SOURCES:
-            logger.info("DEBUG_SKIP_SOURCES: skipping %s (%s)", source_label, link)
-            idx += 1
-            continue
-
-        # Skip source types that have failed too many times
-        if source_failures.get(source_label, 0) >= SOURCE_FAILURE_THRESHOLD:
-            logger.info("Skipping %s - source type '%s' failed %d times", link, source_label, SOURCE_FAILURE_THRESHOLD)
-            idx += 1
-            continue
-        
-        try:
-            current_pos = idx + 1
-            # Update total if we added more sources
-            total_sources = len(links_queue)
-            
-            logger.info("Trying download source [%s]: %s (%d/%d)", source_label, link, current_pos, total_sources)
-            
-            # Build source context for status messages (e.g., "Welib (1/12)")
-            source_context = f"{friendly_name} (Server #{current_pos})"
-            
-            # Update status with simple message showing which source we're trying
-            if status_callback:
-                status_callback("resolving", f"Trying {source_context}")
-
-            download_url = _get_download_url(link, book_info.title, cancel_flag, status_callback, selector, source_context)
-            if download_url == "":
-                raise Exception("No download URL resolved")
-
-            logger.info("Resolved download URL [%s]: %s", source_label, download_url)
-
-            # Pass source page as referer (required by some sites)
-            data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag, selector, status_callback, referer=link)
-            if not data:
-                raise Exception("No data received from download")
-            
-            # Validate file size - reject suspiciously small files
-            file_size = data.tell()
-            if file_size < MIN_VALID_FILE_SIZE:
-                logger.warning(f"Downloaded file too small ({file_size} bytes), likely an error page")
-                raise Exception(f"File too small ({file_size} bytes)")
-
-            logger.debug(f"Download finished ({file_size} bytes). Writing to {book_path}")
-            data.seek(0)  # Reset buffer position before writing
-            with open(book_path, "wb") as f:
-                f.write(data.getbuffer())
-            return download_url
-
-        except Exception as e:
-            logger.warning(f"Failed to download from {link} (source={source_label}): {e}")
-            source_failures[source_label] = source_failures.get(source_label, 0) + 1
-            idx += 1
-            # If we exhausted primary links and haven't loaded welib yet, fetch them lazily
-            if (
-                idx >= len(links_queue)
-                and not welib_fallback_loaded
-                and USE_CF_BYPASS
-                and ALLOW_USE_WELIB
-            ):
-                welib_selector = selector  # reuse AA mirror selector for consistency
-                welib_links = _get_download_urls_from_welib(book_info.id, selector=welib_selector, cancel_flag=cancel_flag)
-                welib_fallback_loaded = True
-                if welib_links:
-                    new_links = [wl for wl in welib_links if wl not in links_queue]
-                    if new_links:
-                        logger.info("Adding welib fallback links (%d)", len(new_links))
-                        links_queue.extend(new_links)
-                        # continue loop to try newly added links
-            continue
-
-    # All sources exhausted - report final error to UI
-    if status_callback:
-        status_callback("error", f"All {len(links_queue)} sources failed")
-    
-    return None
-
-
-def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None, selector: Optional[network.AAMirrorSelector] = None, source_context: Optional[str] = None) -> str:
-    """Extract actual download URL from various source pages.
-    
-    Args:
-        link: URL to extract download link from
-        title: Book title for logging
-        cancel_flag: Optional cancellation flag
-        status_callback: Optional callback for status updates
-        selector: Optional AA mirror selector
-        source_context: Optional context string like "Welib (1/12)" for status messages
-    """
-    sel = selector or network.AAMirrorSelector()
-
-    # AA fast download API (JSON response)
-    if link.startswith(f"{network.get_aa_base_url()}/dyn/api/fast_download.json"):
-        page = downloader.html_get_page(link, selector=sel, cancel_flag=cancel_flag)
-        return downloader.get_absolute_url(link, json.loads(page).get("download_url", ""))
-
-    html = downloader.html_get_page(link, selector=sel, cancel_flag=cancel_flag)
-    if not html:
-        return ""
-
-    soup = BeautifulSoup(html, "html.parser")
-    url = ""
-
-    # Z-Library
-    if link.startswith("https://z-lib."):
-        dl = soup.find("a", href=True, class_="addDownloadedBook")
-        url = dl["href"] if dl else ""
-
-    # AA slow download / partner servers
-    elif "/slow_download/" in link:
-        url = _extract_slow_download_url(soup, link, title, cancel_flag, status_callback, sel, source_context)
-
-    # Libgen (GET button)
-    else:
-        get_btn = soup.find("a", string="GET")
-        url = get_btn["href"] if get_btn else ""
-
-    return downloader.get_absolute_url(link, url)
-
-
-def _extract_slow_download_url(soup: BeautifulSoup, link: str, title: str, cancel_flag: Optional[Event], status_callback, selector, source_context: Optional[str] = None) -> str:
-    """Extract download URL from AA slow download pages."""
-    # Try "Download now" button variations
-    dl_link = soup.find("a", href=True, string="📚 Download now")
-    if not dl_link:
-        dl_link = soup.find("a", href=True, string=lambda s: s and "Download now" in s)
-    if dl_link:
-        return dl_link["href"]
-
-    # Try finding URL in gray background span (AA's copy URL format)
-    # The URL appears as plain text in <span class="bg-gray-200 ...">http://...</span>
-    for span in soup.find_all("span", class_=lambda c: c and "bg-gray-200" in c):
-        text = span.get_text(strip=True)
-        if text.startswith("http://") or text.startswith("https://"):
-            return text
-
-    # Try "copy this URL" pattern (legacy)
-    copy_text = soup.find(string=lambda s: s and "copy this url" in s.lower())
-    if copy_text and copy_text.parent:
-        parent = copy_text.parent
-        next_link = parent.find_next("a", href=True)
-        if next_link and next_link.get("href"):
-            return next_link["href"]
-        code_elem = parent.find_next("code")
-        if code_elem:
-            return code_elem.get_text(strip=True)
-        for sibling in parent.find_next_siblings():
-            text = sibling.get_text(strip=True) if hasattr(sibling, 'get_text') else str(sibling).strip()
-            if text.startswith("http"):
-                return text
-
-    # Check for countdown timer (waitlist)
-    countdown = soup.find("span", class_="js-partner-countdown")
-    if countdown:
-        # Cap countdown at 10 minutes to prevent malformed HTML from blocking indefinitely
-        MAX_COUNTDOWN_SECONDS = 600
-        try:
-            raw_countdown = int(countdown.text)
-        except (ValueError, TypeError):
-            logger.warning(f"Invalid countdown value '{countdown.text}', skipping wait")
-            raw_countdown = 0
-        sleep_time = min(raw_countdown, MAX_COUNTDOWN_SECONDS)
-        if raw_countdown > MAX_COUNTDOWN_SECONDS:
-            logger.warning(f"Countdown {raw_countdown}s exceeds max, capping at {MAX_COUNTDOWN_SECONDS}s")
-        logger.info(f"Waiting {sleep_time}s for {title}")
-        
-        # Live countdown with status updates
-        remaining = sleep_time
-        while remaining > 0:
-            # Format countdown message with source context
-            if source_context:
-                wait_msg = f"{source_context} - Waiting {remaining}s"
-            else:
-                wait_msg = f"Waiting {remaining}s"
-            
-            if status_callback:
-                status_callback("resolving", wait_msg)
-            
-            # Wait 1 second (or until cancelled)
-            if cancel_flag and cancel_flag.wait(timeout=1):
-                logger.info(f"Cancelled wait for {title}")
-                return ""
-            
-            remaining -= 1
-        
-        # After countdown, update status and re-fetch
-        if status_callback and source_context:
-            status_callback("resolving", f"{source_context} - Fetching...")
-        
-        return _get_download_url(link, title, cancel_flag, status_callback, selector, source_context)
-
-    # Debug fallback
-    link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
-    logger.warning(f"No download URL found. First 10 links: {link_texts}")
-    return ""

cloudflare_bypasser_external.py

@@ -1,159 +0,0 @@
-from logger import setup_logger
-from threading import Event
-from typing import Optional, TYPE_CHECKING
-import requests
-import time
-import random
-
-if TYPE_CHECKING:
-    import network
-
-
-class BypassCancelledException(Exception):
-    """Raised when a bypass operation is cancelled."""
-    pass
-
-try:
-    from env import EXT_BYPASSER_PATH, EXT_BYPASSER_TIMEOUT, EXT_BYPASSER_URL
-except ImportError:
-    raise RuntimeError("Failed to import environment variables. Are you using an `extbp` image?")
-
-logger = setup_logger(__name__)
-
-# Connection timeout (seconds) - how long to wait for external bypasser to accept connection
-CONNECT_TIMEOUT = 10
-# Maximum read timeout cap (seconds) - hard limit regardless of EXT_BYPASSER_TIMEOUT
-MAX_READ_TIMEOUT = 120
-# Buffer added to bypasser's configured timeout (seconds) - accounts for processing overhead
-READ_TIMEOUT_BUFFER = 15
-# Retry settings for bypasser failures
-MAX_RETRY = 5
-BACKOFF_BASE = 1.0
-BACKOFF_CAP = 10.0
-
-
-def _fetch_via_bypasser(target_url: str) -> Optional[str]:
-    """Make a single request to the external bypasser service.
-    
-    Args:
-        target_url: The URL to fetch through the bypasser
-        
-    Returns:
-        HTML content if successful, None otherwise
-    """
-    if not EXT_BYPASSER_URL or not EXT_BYPASSER_PATH:
-        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
-        return None
-    
-    bypasser_endpoint = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}"
-    headers = {"Content-Type": "application/json"}
-    payload = {
-        "cmd": "request.get",
-        "url": target_url,
-        "maxTimeout": EXT_BYPASSER_TIMEOUT
-    }
-    
-    # Calculate read timeout: bypasser timeout (ms → s) + buffer, capped at max
-    read_timeout = min((EXT_BYPASSER_TIMEOUT / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
-    
-    try:
-        response = requests.post(
-            bypasser_endpoint, 
-            headers=headers, 
-            json=payload, 
-            timeout=(CONNECT_TIMEOUT, read_timeout)
-        )
-        response.raise_for_status()
-        result = response.json()
-        
-        status = result.get('status', 'unknown')
-        message = result.get('message', '')
-        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
-        
-        # Check for error status (bypasser returns status="error" with solution=null on failure)
-        if status != 'ok':
-            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
-            return None
-        
-        solution = result.get('solution')
-        if not solution:
-            logger.warning(f"External bypasser returned empty solution for '{target_url}'")
-            return None
-        
-        html = solution.get('response', '')
-        if not html:
-            logger.warning(f"External bypasser returned empty response for '{target_url}'")
-            return None
-        
-        return html
-        
-    except requests.exceptions.Timeout:
-        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
-        return None
-    except requests.exceptions.RequestException as e:
-        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
-        return None
-    except (KeyError, TypeError, ValueError) as e:
-        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
-        return None
-
-
-def get_bypassed_page(url: str, selector: Optional["network.AAMirrorSelector"] = None, cancel_flag: Optional[Event] = None) -> Optional[str]:
-    """Fetch HTML content from a URL using an external Cloudflare bypasser service.
-
-    Retries with exponential backoff and mirror/DNS rotation on failure.
-
-    Args:
-        url: Target URL to fetch
-        selector: Mirror selector for AA URL rewriting and rotation
-        cancel_flag: Optional threading Event to signal cancellation
-
-    Returns:
-        HTML content if successful, None otherwise
-
-    Raises:
-        BypassCancelledException: If cancel_flag is set during operation
-    """
-    import network
-    sel = selector or network.AAMirrorSelector()
-
-    for attempt in range(1, MAX_RETRY + 1):
-        # Check for cancellation before each attempt
-        if cancel_flag and cancel_flag.is_set():
-            logger.info("External bypasser cancelled by user")
-            raise BypassCancelledException("Bypass cancelled")
-
-        attempt_url = sel.rewrite(url)
-        result = _fetch_via_bypasser(attempt_url)
-        if result:
-            return result
-
-        if attempt == MAX_RETRY:
-            break
-
-        # Check for cancellation before backoff wait
-        if cancel_flag and cancel_flag.is_set():
-            logger.info("External bypasser cancelled during retry")
-            raise BypassCancelledException("Bypass cancelled")
-
-        # Backoff with jitter before retry, checking cancellation during wait
-        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
-        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
-
-        # Check cancellation during delay (check every second)
-        for _ in range(int(delay)):
-            if cancel_flag and cancel_flag.is_set():
-                logger.info("External bypasser cancelled during backoff")
-                raise BypassCancelledException("Bypass cancelled")
-            time.sleep(1)
-        # Sleep remaining fraction
-        remaining = delay - int(delay)
-        if remaining > 0:
-            time.sleep(remaining)
-
-        # Rotate mirror/DNS for next attempt
-        new_base, action = sel.next_mirror_or_rotate_dns()
-        if action in ("mirror", "dns") and new_base:
-            logger.info(f"Rotated {action} for retry")
-
-    return None

compose/docker-compose.lite.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark-lite:
+    image: ghcr.io/calibrain/shelfmark-lite:latest
+    environment:
+      # EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.tor.yml

@@ -0,0 +1,20 @@
+# Routes all traffic through Tor - requires NET_ADMIN capability
+services:
+  shelfmark-tor:
+    image: ghcr.io/calibrain/shelfmark:latest
+    environment:
+      FLASK_PORT: 8084
+      USING_TOR: true
+      PUID: 1000
+      PGID: 1000
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    container_name: shelfmark
+    environment:
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

config.py

@@ -1,122 +0,0 @@
-"""Configuration settings for the book downloader application."""
-
-import os
-from pathlib import Path
-import json
-import env
-from logger import setup_logger
-
-logger = setup_logger(__name__)
-
-for key, value in env.__dict__.items():
-    if not key.startswith('_'):
-        if key == "AA_DONATOR_KEY" and value.strip() != "":
-            value = "REDACTED"
-        logger.info(f"{key}: {value}")
-
-with open("data/book-languages.json") as file:
-    _SUPPORTED_BOOK_LANGUAGE = json.load(file)
-
-# Directory settings
-BASE_DIR = Path(__file__).resolve().parent
-logger.info(f"BASE_DIR: {BASE_DIR}")
-if env.ENABLE_LOGGING:
-    env.LOG_DIR.mkdir(exist_ok=True)
-
-# Create necessary directories
-env.TMP_DIR.mkdir(exist_ok=True)
-env.INGEST_DIR.mkdir(exist_ok=True)
-
-CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
-logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
-logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
-logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
-
-# Network settings
-_custom_dns = env._CUSTOM_DNS.lower().strip()
-_doh_server = ""
-
-if _custom_dns == "auto" or _custom_dns == "":
-    # Auto mode - DNS provider rotation handled by network.py
-    # Starts with system DNS, switches to providers from DNS_PROVIDERS on failure
-    CUSTOM_DNS = []
-    _doh_server = ""
-    logger.info("CUSTOM_DNS: auto (starts with system DNS, rotates on failure)")
-elif _custom_dns == "google":
-    CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860:0000:0000:0000:0000:8888", "2001:4860:4860:0000:0000:0000:0000:8844"]
-    _doh_server = "https://dns.google/resolve"
-    logger.info(f"CUSTOM_DNS: google {CUSTOM_DNS}")
-elif _custom_dns == "quad9":
-    CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:00fe:0000:0000:0000:0000:0000:00fe", "2620:00fe:0000:0000:0000:0000:0000:0009"]
-    _doh_server = "https://dns.quad9.net/dns-query"
-    logger.info(f"CUSTOM_DNS: quad9 {CUSTOM_DNS}")
-elif _custom_dns == "cloudflare":
-    CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700:0000:0000:0000:0000:1111", "2606:4700:4700:0000:0000:0000:0000:1001"]
-    _doh_server = "https://cloudflare-dns.com/dns-query"
-    logger.info(f"CUSTOM_DNS: cloudflare {CUSTOM_DNS}")
-elif _custom_dns == "opendns":
-    CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:0119:0035:0000:0000:0000:0000:0035", "2620:0119:0053:0000:0000:0000:0000:0053"]
-    _doh_server = "https://doh.opendns.com/dns-query"
-    logger.info(f"CUSTOM_DNS: opendns {CUSTOM_DNS}")
-else:
-    # Custom DNS IPs provided by user
-    _custom_dns_ip = _custom_dns.split(",")
-    CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()]
-    logger.info(f"CUSTOM_DNS: custom {CUSTOM_DNS}")
-DOH_SERVER = _doh_server
-if env.USE_DOH:
-    DOH_SERVER = _doh_server
-else:
-    DOH_SERVER = ""
-logger.info(f"DOH_SERVER: {DOH_SERVER}")
-
-# Warn about external bypasser DNS limitations
-if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS:
-    logger.warning(
-        "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
-        "not this application's custom DNS settings. If you experience DNS-related blocks, "
-        "configure DNS at the Docker/system level for your FlareSolverr container, "
-        "or consider using the internal bypasser which integrates with the app's DNS system."
-    )
-
-# Proxy settings
-PROXIES = {}
-if env.HTTP_PROXY:
-    PROXIES["http"] = env.HTTP_PROXY
-if env.HTTPS_PROXY:
-    PROXIES["https"] = env.HTTPS_PROXY
-logger.info(f"PROXIES: {PROXIES}")
-
-# Anna's Archive settings
-AA_BASE_URL = env._AA_BASE_URL
-AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
-AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
-AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
-
-# File format settings
-SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
-logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
-
-# Complex language processing logic kept in config.py
-BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
-BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
-if len(BOOK_LANGUAGE) == 0:
-    BOOK_LANGUAGE = ['en']
-
-# Custom script settings with validation logic
-CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
-if CUSTOM_SCRIPT:
-    if not os.path.exists(CUSTOM_SCRIPT):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
-        CUSTOM_SCRIPT = ""
-    elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
-        CUSTOM_SCRIPT = ""
-
-# Debugging settings
-if not env.USING_EXTERNAL_BYPASSER:
-    # Virtual display settings for debugging internal cloudflare bypasser
-    VIRTUAL_SCREEN_SIZE = (1024, 768)
-    RECORDING_DIR = env.LOG_DIR / "recording"
-    if env.DEBUG:
-        RECORDING_DIR.mkdir(parents=True, exist_ok=True)

docker-compose.dev.extbp.yml

@@ -0,0 +1,25 @@
+# Local development - External bypasser variant (lite)
+services:
+  shelfmark-lite-dev:
+    extends:
+      file: ./compose/docker-compose.lite.yml
+      service: shelfmark-lite
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark-lite
+    environment:
+      DEBUG: true
+      EXT_BYPASSER_URL: http://flaresolverr:8191
+      EXT_BYPASSER_PATH: /v1
+      EXT_BYPASSER_TIMEOUT: 60000
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads
+
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.dev.tor.yml

@@ -0,0 +1,20 @@
+# Local development - Tor variant
+services:
+  shelfmark-tor-dev:
+    extends:
+      file: ./compose/docker-compose.tor.yml
+      service: shelfmark-tor
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    environment:
+      DEBUG: true
+      USING_TOR: true
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.dev.yml

@@ -1,14 +1,22 @@
+# Local development - builds from source with debug enabled
 services:
-  calibre-web-automated-book-downloader-dev:
+  shelfmark-dev:
     extends:
-      file: ./docker-compose.yml
-      service: calibre-web-automated-book-downloader
+      file: ./compose/docker-compose.yml
+      service: shelfmark
     build:
       context: .
       dockerfile: Dockerfile
-      target: cwa-bd
+      target: shelfmark
+    cap_add:
+      - SYS_PTRACE
     environment:
       DEBUG: true
     volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      - ./shelfmark:/app/shelfmark:ro
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.extbp.dev.yml

@@ -1,34 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-extbp-dev:
-    container_name: cwa-bd-extbp-dev
-    extends:
-      file: ./docker-compose.extbp.yml
-      service: calibre-web-automated-book-downloader-extbp
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-extbp
-    environment:
-      DEBUG: true
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
-      USE_CF_BYPASS: true  # Enable Cloudflare bypass (default: true)
-      # External Cloudflare Bypass environment variables
-      EXT_BYPASSER_URL: "http://flaresolverr:8191"  # URL of the external Cloudflare resolver service (used FlareSolverr)
-      EXT_BYPASSER_PATH: "/v1"  # Path for external Cloudflare resolver API (default: /v1)
-      EXT_BYPASSER_TIMEOUT: 60000  # Timeout for external Cloudflare resolver requests (default: 60000)
-    volumes:
-      #- /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      #- /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader
-      - ./deploy/ingest:/cwa-book-ingest
-      - ./deploy/log:/var/log/cwa-book-downloader
-      - ./deploy/tmp:/tmp/cwa-book-downloader
-
-  flaresolverr:  # External Cloudflare resolver service
-    image: ghcr.io/flaresolverr/flaresolverr:v3.3.22
-    container_name: flaresolverr
-    environment:
-      LOG_LEVEL: info
-      LOG_HTML: false
-      CAPTCHA_SOLVER: none
-      TZ: Europe/Rome

docker-compose.extbp.yml

@@ -1,27 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-extbp:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-extbp:latest
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      UID: 1000
-      GID: 100
-      # CWA_DB_PATH: /auth/app.db  # Uncomment to enable authentication
-      # SESSION_COOKIE_SECURE: 'true'  # Set to 'true' if accessing ONLY via HTTPS
-      # DEBUG: 'true'  # Enable debug mode (debug button, verbose logging)
-      EXT_BYPASSER_URL: http://flaresolverr:8191
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above)
-      #- /cwa/config/path/app.db:/auth/app.db:ro
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.test-clients.yml

@@ -0,0 +1,181 @@
+# Test stack for download client development
+# Includes shelfmark + all download clients on same network with shared volumes
+#
+# Usage:
+#   docker compose -f docker-compose.test-clients.yml up -d
+#   # Access shelfmark at http://localhost:8084
+#   # Configure clients in Settings > Prowlarr > Download Clients
+#
+# Web UIs:
+#   - shelfmark:    http://localhost:8084
+#   - Prowlarr:     http://localhost:9696 (no auth by default)
+#   - qBittorrent:  http://localhost:8080 (check container logs for temp password)
+#   - Transmission: http://localhost:9091 (admin / admin)
+#   - Deluge:       http://localhost:8112 (password: deluge)
+#   - NZBGet:       http://localhost:6789 (nzbget / tegbzn6789)
+#   - SABnzbd:      http://localhost:8085 (complete setup wizard for API key)
+#   - rTorrent:     http://localhost:8000 (admin / admin - if auth enabled)
+#
+
+services:
+  shelfmark:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    container_name: test-shelfmark
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      # All client configuration is done via Settings UI
+      # Use Docker service names for URLs:
+      #   - qBittorrent: http://qbittorrent:8080
+      #   - Transmission: http://transmission:9091
+      #   - Deluge Web UI: http://deluge:8112
+      #   - NZBGet: http://nzbget:6789
+      #   - SABnzbd: http://sabnzbd:8080
+      #   - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
+    ports:
+      - "8084:8084"
+    volumes:
+      # Config and state
+      - ./.local/test-clients/shelfmark/config:/config
+      - ./.local/test-clients/shelfmark/log:/var/log/shelfmark
+      # Book destination directory (where completed books go)
+      - ./.local/test-clients/books:/books
+      # Staging directory
+      - ./.local/test-clients/tmp:/tmp/shelfmark
+      # CRITICAL: Mount client download directories so shelfmark can access completed files
+      - ./.local/test-clients/downloads:/downloads
+      # Mount source code for hot-reload (no rebuild needed for Python changes)
+      - ./shelfmark:/app/shelfmark:ro
+      # Mount tests for running pytest in container
+      - ./tests:/app/tests:ro
+      - ./pyproject.toml:/app/pyproject.toml:ro
+      # Mount client configs for integration tests to read credentials
+      - ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
+      - ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
+    depends_on:
+      - nzbget
+      - sabnzbd
+      - qbittorrent
+      - transmission
+      - deluge
+      - rtorrent
+    restart: unless-stopped
+
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:latest
+    container_name: test-prowlarr
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/prowlarr/config:/config
+    ports:
+      - "9696:9696"
+    restart: unless-stopped
+
+  nzbget:
+    image: lscr.io/linuxserver/nzbget:latest
+    container_name: test-nzbget
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/nzbget/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/nzbget/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "6789:6789"  # Web UI / JSON-RPC
+    restart: unless-stopped
+
+  sabnzbd:
+    image: lscr.io/linuxserver/sabnzbd:latest
+    container_name: test-sabnzbd
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/sabnzbd/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8085:8080"  # Web UI (external:internal)
+    restart: unless-stopped
+
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: test-qbittorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - WEBUI_PORT=8080
+    volumes:
+      - ./.local/test-clients/qbittorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/qbittorrent/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "8080:8080"  # Web UI / API
+      - "6882:6881"
+      - "6882:6881/udp"
+    restart: unless-stopped
+
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: test-transmission
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - USER=admin
+      - PASS=admin
+    volumes:
+      - ./.local/test-clients/transmission/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "9091:9091"  # Web UI / RPC
+      - "51413:51413"
+      - "51413:51413/udp"
+    restart: unless-stopped
+
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: test-deluge
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - DELUGE_LOGLEVEL=error
+    volumes:
+      - ./.local/test-clients/deluge/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8112:8112"  # Web UI
+      - "58846:58846"  # Daemon RPC
+      - "6881:6881"
+      - "6881:6881/udp"
+    restart: unless-stopped
+
+  rtorrent:
+    image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
+    container_name: test-rtorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/rtorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8000:8000"  # XMLRPC
+      - "8089:8080"  # ruTorrent Web UI
+      - "9000:9000"  # SCGI port
+      - "50000:50000"  # Incoming connections
+      - "6881:6881/udp"
+    restart: unless-stopped

docker-compose.tor.dev.yml

@@ -1,14 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor-dev:
-    extends:
-      file: ./docker-compose.tor.yml
-      service: calibre-web-automated-book-downloader-tor
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-tor
-    environment:
-      DEBUG: true
-    volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader

docker-compose.tor.yml

@@ -1,26 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      USING_TOR: true
-      # CWA_DB_PATH: /auth/app.db  # Uncomment to enable authentication
-      # SESSION_COOKIE_SECURE: 'true'  # Set to 'true' if accessing ONLY via HTTPS
-      # DEBUG: 'true'  # Enable debug mode (debug button, verbose logging)
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be 
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above)
-      #- /cwa/config/path/app.db:/auth/app.db:ro

docker-compose.yml

@@ -1,32 +0,0 @@
-services:
-  calibre-web-automated-book-downloader:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
-    # Uncomment to build the image from the Dockerfile for local testing changes.
-    # Remember to comment out the image line above.
-    #build: .
-    container_name: calibre-web-automated-book-downloader
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      UID: 1000
-      GID: 100
-      # CWA_DB_PATH: /auth/app.db  # Uncomment to enable authentication (also uncomment volume below)
-      # CALIBRE_WEB_URL: http://localhost:8080 # Uncomment and add your custom library URL to enable "Go To Library" button in the Web UI
-      # SESSION_COOKIE_SECURE: 'true'  # Set to 'true' if accessing ONLY via HTTPS
-      # DEBUG: 'true'  # Enable debug mode (debug button, verbose logging)
-      # Queue management settings
-      MAX_CONCURRENT_DOWNLOADS: 3
-      DOWNLOAD_PROGRESS_UPDATE_INTERVAL: 5
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above)
-      #- /cwa/config/path/app.db:/auth/app.db:ro

docs/configuration.md

@@ -0,0 +1,3 @@
+# Configuration
+
+TODO

docs/dev/index.md

@@ -0,0 +1,3 @@
+# Developer Documentation
+
+TODO

docs/dev/plugin-settings.md

@@ -0,0 +1,628 @@
+# Plugin Settings Integration Guide
+
+This guide explains how to add configuration settings to plugins (Metadata Providers and Release Sources) so they appear in the Settings UI.
+
+## Overview
+
+The settings system uses a decorator-based registration pattern. Plugins register their settings when their module is imported, and the frontend dynamically renders the appropriate UI based on the schema provided by the backend.
+
+**Key features:**
+- Settings are defined in Python and automatically rendered in the React frontend
+- Values persist across container restarts via JSON config files
+- Changes take effect immediately without restart (unless marked otherwise)
+
+## Quick Start
+
+Add settings to your plugin in 3 steps:
+
+```python
+from shelfmark.core.settings_registry import (
+    register_settings,
+    TextField,
+    PasswordField,
+    ActionButton,
+)
+
+@register_settings(
+    name="my_plugin",           # Unique identifier
+    display_name="My Plugin",   # Shown in sidebar
+    icon="wrench",              # Icon name
+    order=100,                  # Sort order (lower = higher in list)
+    group="metadata_providers"  # Optional: group in sidebar
+)
+def my_plugin_settings():
+    return [
+        PasswordField(
+            key="MY_PLUGIN_API_KEY",
+            label="API Key",
+            description="Your API key from the provider",
+            required=True,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+def _test_connection():
+    # Perform connection test
+    return {"success": True, "message": "Connected successfully!"}
+```
+
+## Available Field Types
+
+### TextField
+
+Single-line text input for strings.
+
+```python
+TextField(
+    key="MY_SETTING",           # Config key
+    label="Setting Name",       # Display label
+    description="Help text",    # Optional description below field
+    default="",                 # Default value
+    placeholder="Enter value",  # Placeholder text
+    max_length=100,             # Optional max characters
+    required=False,             # Is this field required?
+    requires_restart=False,     # Does changing this need a restart?
+    show_when=None,             # Conditional visibility (see below)
+    disabled_when=None,         # Conditional disable (see below)
+)
+```
+
+### PasswordField
+
+Masked input for sensitive values (API keys, passwords). Values are never echoed back to the frontend.
+
+```python
+PasswordField(
+    key="API_KEY",
+    label="API Key",
+    description="Your secret API key",
+    placeholder="sk-...",
+    required=True,
+)
+```
+
+### NumberField
+
+Numeric input with optional min/max constraints.
+
+```python
+NumberField(
+    key="TIMEOUT",
+    label="Timeout (seconds)",
+    description="Connection timeout in seconds",
+    default=30,
+    min_value=5,
+    max_value=300,
+    step=1,                     # Increment step
+    required=False,
+)
+```
+
+### CheckboxField
+
+Toggle switch for boolean values.
+
+```python
+CheckboxField(
+    key="ENABLE_FEATURE",
+    label="Enable Feature",
+    description="Turn this feature on or off",
+    default=False,
+)
+```
+
+### SelectField
+
+Dropdown for single-choice selection.
+
+```python
+SelectField(
+    key="LOG_LEVEL",
+    label="Log Level",
+    description="Logging verbosity",
+    default="info",
+    options=[
+        {"value": "debug", "label": "Debug"},
+        {"value": "info", "label": "Info"},
+        {"value": "warning", "label": "Warning"},
+        {"value": "error", "label": "Error"},
+    ],
+)
+```
+
+### MultiSelectField
+
+Multi-choice selection from a list of options.
+
+```python
+MultiSelectField(
+    key="SUPPORTED_FORMATS",
+    label="Supported Formats",
+    description="Select which formats to support",
+    default=["epub", "mobi"],
+    options=[
+        {"value": "epub", "label": "EPUB"},
+        {"value": "mobi", "label": "MOBI"},
+        {"value": "pdf", "label": "PDF"},
+        {"value": "azw3", "label": "AZW3"},
+    ],
+)
+```
+
+### ActionButton
+
+Button that executes a callback function. Does not store a value.
+
+```python
+ActionButton(
+    key="test_connection",      # Unique key for the action
+    label="Test Connection",    # Button text
+    description="Test the API connection",
+    style="primary",            # "default", "primary", or "danger"
+    callback=my_callback_fn,    # Function to execute
+)
+
+def my_callback_fn():
+    """Callback must return dict with 'success' and 'message' keys."""
+    try:
+        # Perform action
+        return {"success": True, "message": "Connection successful!"}
+    except Exception as e:
+        return {"success": False, "message": f"Failed: {str(e)}"}
+```
+
+### HeadingField
+
+Display-only section heading with optional link. Does not store a value.
+
+```python
+HeadingField(
+    key="section_heading",      # Unique key
+    title="Configuration",      # Heading text
+    description="Configure the plugin settings below",
+    link_url="https://example.com/docs",  # Optional link
+    link_text="View Documentation",       # Link text
+)
+```
+
+## Common Field Properties
+
+All field types support these common properties:
+
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `key` | `str` | Required | Unique identifier for this setting |
+| `label` | `str` | Required | Display label in the UI |
+| `description` | `str` | `""` | Help text shown below the field |
+| `default` | `Any` | `None` | Default value if not set |
+| `required` | `bool` | `False` | Whether the field must have a value |
+| `disabled` | `bool` | `False` | Disable the field (greyed out) |
+| `disabled_reason` | `str` | `""` | Explanation shown when disabled |
+| `requires_restart` | `bool` | `False` | Whether changes require container restart |
+| `show_when` | `dict` | `None` | Conditional visibility (see below) |
+| `disabled_when` | `dict` | `None` | Conditional disable (see below) |
+
+## Conditional Visibility
+
+Fields can be shown/hidden based on other field values using `show_when`:
+
+```python
+# Only show DNS servers field when custom DNS is selected
+TextField(
+    key="CUSTOM_DNS_SERVERS",
+    label="DNS Servers",
+    description="Comma-separated DNS server IPs",
+    show_when={"field": "DNS_PROVIDER", "value": "manual"},
+)
+```
+
+The field will only be visible when the referenced field has the specified value.
+
+## Conditional Disable
+
+Fields can be enabled/disabled based on other field values using `disabled_when`:
+
+```python
+# Disable timeout field when feature is disabled
+NumberField(
+    key="FEATURE_TIMEOUT",
+    label="Timeout (seconds)",
+    description="Request timeout",
+    default=30,
+    disabled_when={
+        "field": "FEATURE_ENABLED",
+        "value": False,
+        "reason": "Enable the feature first"
+    },
+)
+```
+
+The field will be greyed out with the specified reason when the condition is met.
+
+## Settings Groups
+
+Register a group to organize related settings tabs in the sidebar:
+
+```python
+from shelfmark.core.settings_registry import register_group
+
+# Register a group (do this once, usually in a central config file)
+register_group(
+    name="my_group",
+    display_name="My Group",
+    icon="folder",
+    order=50,
+)
+
+# Then register settings to the group
+@register_settings(
+    name="plugin_a",
+    display_name="Plugin A",
+    icon="puzzle",
+    order=51,
+    group="my_group",  # Assigns to the group
+)
+def plugin_a_settings():
+    return [...]
+```
+
+**Existing groups:**
+- `direct_download` (order=20): For download-related settings
+- `metadata_providers` (order=50): For metadata provider plugins
+
+## Value Resolution Priority
+
+Settings values are resolved in this order (highest priority first):
+
+1. **Config File** - Stored in `CONFIG_DIR/plugins/<tab_name>.json`
+2. **Field Default** - Value specified in the field definition
+
+The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdirectory.
+
+## Reading Setting Values
+
+Use the `config` singleton to read setting values in your plugin code:
+
+```python
+from shelfmark.core.config import config
+
+# Get a setting value with default fallback
+api_key = config.get("MY_PLUGIN_API_KEY", "")
+timeout = config.get("MY_PLUGIN_TIMEOUT", 30)
+
+# Or access as attributes (raises AttributeError if not found)
+api_key = config.MY_PLUGIN_API_KEY
+
+# Check all cached settings
+all_settings = config.get_all()
+```
+
+The config singleton:
+- Automatically resolves values from config files with field defaults as fallback
+- Caches values for performance
+- Refreshes automatically when settings are updated via the UI
+
+## Complete Example: Metadata Provider
+
+Here's a complete example for a metadata provider plugin:
+
+```python
+# shelfmark/metadata_providers/my_provider.py
+
+from shelfmark.metadata_providers.base import (
+    MetadataProvider,
+    register_provider,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_connection():
+    """Test API connection callback."""
+    api_key = config.get("MY_PROVIDER_API_KEY", "")
+    if not api_key:
+        return {"success": False, "message": "API key not configured"}
+
+    try:
+        # Perform actual connection test
+        # response = requests.get(...)
+        return {"success": True, "message": "Connected to My Provider API"}
+    except Exception as e:
+        return {"success": False, "message": f"Connection failed: {str(e)}"}
+
+
+@register_settings(
+    name="my_provider",
+    display_name="My Provider",
+    icon="book",
+    order=53,
+    group="metadata_providers",
+)
+def my_provider_settings():
+    """Define settings for this metadata provider."""
+    return [
+        HeadingField(
+            key="my_provider_heading",
+            title="My Provider",
+            description="A metadata provider for book information",
+            link_url="https://myprovider.com",
+            link_text="Visit My Provider",
+        ),
+        PasswordField(
+            key="MY_PROVIDER_API_KEY",
+            label="API Key",
+            description="Your My Provider API key",
+            placeholder="Enter your API key",
+            required=True,
+        ),
+        CheckboxField(
+            key="MY_PROVIDER_INCLUDE_COVERS",
+            label="Include Cover Images",
+            description="Fetch cover images when searching",
+            default=True,
+        ),
+        TextField(
+            key="MY_PROVIDER_BASE_URL",
+            label="API Base URL",
+            description="Override the default API endpoint",
+            default="https://api.myprovider.com/v1",
+            required=False,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            description="Verify your API key works",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+
+@register_provider("my_provider")
+class MyProvider(MetadataProvider):
+    """My Provider metadata implementation."""
+
+    name = "my_provider"
+    display_name = "My Provider"
+    requires_auth = True
+
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
+        self.base_url = config.get(
+            "MY_PROVIDER_BASE_URL",
+            "https://api.myprovider.com/v1"
+        )
+
+    def is_available(self) -> bool:
+        return bool(self.api_key)
+
+    def search(self, query: str):
+        # Implementation...
+        pass
+
+    def get_book(self, book_id: str):
+        # Implementation...
+        pass
+```
+
+## Complete Example: Release Source
+
+Here's a complete example for a release source plugin:
+
+```python
+# shelfmark/release_sources/my_source.py
+
+from shelfmark.release_sources.base import (
+    ReleaseSource,
+    DownloadHandler,
+    register_source,
+    register_handler,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    NumberField,
+    CheckboxField,
+    SelectField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_source():
+    """Test source availability callback."""
+    base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+    try:
+        # Test connectivity
+        return {"success": True, "message": f"Source available at {base_url}"}
+    except Exception as e:
+        return {"success": False, "message": f"Source unavailable: {str(e)}"}
+
+
+@register_settings(
+    name="my_source",
+    display_name="My Source",
+    icon="download",
+    order=25,
+    group="direct_download",
+)
+def my_source_settings():
+    """Define settings for this release source."""
+    return [
+        HeadingField(
+            key="my_source_heading",
+            title="My Source Configuration",
+            description="Configure the My Source download provider",
+        ),
+        CheckboxField(
+            key="MY_SOURCE_ENABLED",
+            label="Enable My Source",
+            description="Include My Source in download fallback chain",
+            default=True,
+        ),
+        TextField(
+            key="MY_SOURCE_URL",
+            label="Source URL",
+            description="Base URL for the source",
+            default="https://mysource.com",
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        NumberField(
+            key="MY_SOURCE_TIMEOUT",
+            label="Timeout (seconds)",
+            description="Request timeout",
+            default=30,
+            min_value=10,
+            max_value=120,
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        SelectField(
+            key="MY_SOURCE_PRIORITY",
+            label="Priority",
+            description="Where in the fallback chain to try this source",
+            default="normal",
+            options=[
+                {"value": "high", "label": "High (try first)"},
+                {"value": "normal", "label": "Normal"},
+                {"value": "low", "label": "Low (try last)"},
+            ],
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        ActionButton(
+            key="test_source",
+            label="Test Source",
+            description="Check if the source is accessible",
+            style="primary",
+            callback=_test_source,
+        ),
+    ]
+
+
+@register_source("my_source")
+class MySource(ReleaseSource):
+    """My Source release source implementation."""
+
+    name = "my_source"
+    display_name = "My Source"
+
+    def __init__(self):
+        self.enabled = config.get("MY_SOURCE_ENABLED", True)
+        self.base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+        self.timeout = config.get("MY_SOURCE_TIMEOUT", 30)
+
+    def is_available(self) -> bool:
+        return self.enabled
+
+    def search(self, book):
+        # Implementation...
+        pass
+
+
+@register_handler("my_source")
+class MySourceHandler(DownloadHandler):
+    """Handler for downloading from My Source."""
+
+    name = "my_source"
+
+    def download(self, release, output_path):
+        # Implementation...
+        pass
+```
+
+## Best Practices
+
+1. **Use descriptive keys**: Keys should be uppercase and prefixed with your plugin name (e.g., `MY_PLUGIN_API_KEY`)
+
+2. **Provide helpful descriptions**: Include enough detail in descriptions to help users understand what each setting does
+
+3. **Set sensible defaults**: Users should be able to get started without configuring everything
+
+4. **Use conditional visibility**: Hide advanced options behind enabling checkboxes to reduce UI clutter
+
+5. **Include a test button**: ActionButtons that test connections help users verify their configuration
+
+6. **Mark restart-required settings**: Use `requires_restart=True` for settings that can't be applied live
+
+7. **Group related settings**: Use HeadingField to visually separate sections, and put plugins in appropriate groups
+
+8. **Handle missing values gracefully**: Always provide fallbacks when reading settings in your code
+
+## API Reference
+
+### Backend Routes
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/api/settings` | Get all settings tabs, groups, and values |
+| GET | `/api/settings/<tab_name>` | Get a specific settings tab |
+| PUT | `/api/settings/<tab_name>` | Update settings for a tab |
+| POST | `/api/settings/<tab_name>/action/<action_key>` | Execute an action button callback |
+
+### Response Format
+
+**GET /api/settings**
+```json
+{
+  "groups": [
+    {"name": "direct_download", "displayName": "Direct Download", "icon": "download", "order": 20}
+  ],
+  "tabs": [
+    {
+      "name": "my_plugin",
+      "displayName": "My Plugin",
+      "icon": "book",
+      "order": 53,
+      "group": "metadata_providers",
+      "fields": [
+        {
+          "type": "password",
+          "key": "MY_PLUGIN_API_KEY",
+          "label": "API Key",
+          "description": "Your API key",
+          "hasValue": true,
+          "value": "",
+          "required": true,
+          "disabled": false,
+          "requiresRestart": false
+        }
+      ]
+    }
+  ]
+}
+```
+
+**PUT /api/settings/<tab_name>**
+```json
+// Request
+{"MY_PLUGIN_API_KEY": "new-value", "MY_PLUGIN_TIMEOUT": 60}
+
+// Response
+{
+  "success": true,
+  "message": "Settings updated",
+  "updated": ["MY_PLUGIN_API_KEY", "MY_PLUGIN_TIMEOUT"],
+  "requiresRestart": false
+}
+```
+
+**POST /api/settings/<tab_name>/action/<action_key>**
+```json
+// Response
+{
+  "success": true,
+  "message": "Connection successful!"
+}
+```

docs/index.md

@@ -0,0 +1,3 @@
+# Shelfmark Documentation
+
+TODO

docs/installation.md

@@ -0,0 +1,3 @@
+# Installation
+
+TODO

docs/reverse-proxy.md

@@ -0,0 +1,35 @@
+# Reverse Proxy & Subpath Hosting
+
+Shelfmark can run behind a reverse proxy at the root path (recommended) or
+under a subpath like `/shelfmark`.
+
+## Subpath setup
+
+1) Set the base path in Shelfmark:
+- UI: Settings → Advanced → Base Path
+- Env var: `URL_BASE=/shelfmark`
+
+2) Configure your reverse proxy to forward the subpath to Shelfmark and
+**strip the prefix** before sending to the backend. The proxy must also allow
+WebSocket upgrades for Socket.IO.
+
+Example (Nginx-style):
+
+```
+location /shelfmark/ {
+  proxy_pass http://shelfmark:8084/;
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection "upgrade";
+}
+```
+
+Notes:
+- Use a trailing slash on the `location` and `proxy_pass` to ensure the
+  `/shelfmark` prefix is removed.
+- Health checks still work at `/api/health` without the subpath.
+
+## Root path setup
+
+If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`),
+leave `URL_BASE` empty. This is the simplest option.

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+# Troubleshooting
+
+TODO

docs/url-search-parameters.md

@@ -0,0 +1,75 @@
+# URL Search Parameters
+
+You can trigger searches directly via URL by adding query parameters. This enables bookmarking searches and sharing links.
+
+## Basic Usage
+
+```
+http://your-server:8084/?q=harry+potter
+```
+
+## Supported Parameters
+
+| Parameter | Description | Example |
+|-----------|-------------|---------|
+| `q` or `query` | Main search query | `/?q=dune` |
+| `author` | Filter by author name | `/?author=frank+herbert` |
+| `title` | Filter by book title | `/?title=foundation` |
+| `isbn` | Filter by ISBN | `/?isbn=978-0747532699` |
+| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
+| `format` | Filter by file format | `/?format=epub` |
+| `content` | Filter by content type | `/?content=fiction` |
+| `sort` | Sort order for results | `/?sort=newest` |
+
+## Multiple Values
+
+Some parameters support multiple values by repeating the parameter:
+
+```
+/?lang=en&lang=de&lang=fr
+/?format=epub&format=mobi&format=azw3
+```
+
+## Examples
+
+**Simple search:**
+```
+/?q=lord+of+the+rings
+```
+
+**Search with author filter:**
+```
+/?q=dune&author=frank+herbert
+```
+
+**Search with format and language:**
+```
+/?q=harry+potter&format=epub&lang=en
+```
+
+**Author search with multiple formats:**
+```
+/?author=stephen+king&format=epub&format=mobi
+```
+
+**Search with sort order:**
+```
+/?q=science+fiction&sort=newest
+```
+
+## Search Mode Behavior
+
+### Direct Download Mode (default)
+
+All parameters are used to filter results from the direct download source.
+
+### Universal Mode
+
+Only `q` and `sort` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
+
+## Notes
+
+- URL parameters are read once on page load
+- The URL is not updated when you perform searches manually
+- Spaces should be encoded as `+` or `%20`
+- Invalid or unknown parameters are silently ignored

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/cwa-book-downloader
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/cwa-bd_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+
+    # Cleanup any existing files or folders in the log directory
+    rm -rf "$LOG_DIR"/*
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,10 +56,16 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
@@ -28,34 +78,57 @@ if [ "$TZ" ]; then
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 fi
 
-# Set UID if not set
-if [ -z "$UID" ]; then
-    UID=1000
+# Determine user ID with proper precedence:
+# 1. PUID (LinuxServer.io standard - recommended)
+# 2. UID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+#
+# Note: $UID is a bash builtin that's always set. We use `printenv` to detect
+# if UID was explicitly set as an environment variable (e.g., via docker-compose).
+if [ -n "$PUID" ]; then
+    RUN_UID="$PUID"
+    echo "Using PUID=$RUN_UID"
+elif printenv UID >/dev/null 2>&1; then
+    RUN_UID="$(printenv UID)"
+    echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)"
+else
+    RUN_UID=1000
+    echo "Using default UID=$RUN_UID"
 fi
 
-# Set GID if not set
-if [ -z "$GID" ]; then
-    GID=100
+# Determine group ID with proper precedence:
+# 1. PGID (LinuxServer.io standard - recommended)
+# 2. GID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+if [ -n "$PGID" ]; then
+    RUN_GID="$PGID"
+    echo "Using PGID=$RUN_GID"
+elif [ -n "$GID" ]; then
+    RUN_GID="$GID"
+    echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)"
+else
+    RUN_GID=1000
+    echo "Using default GID=$RUN_GID"
 fi
 
-if ! getent group "$GID" >/dev/null; then
-    echo "Adding group $GID with name appuser"
-    groupadd -g "$GID" appuser
+if ! getent group "$RUN_GID" >/dev/null; then
+    echo "Adding group $RUN_GID with name appuser"
+    groupadd -g "$RUN_GID" appuser
 fi
 
 # Create user if it doesn't exist
-if ! id -u "$UID" >/dev/null 2>&1; then
-    echo "Adding user $UID with name appuser"
-    useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser
+if ! id -u "$RUN_UID" >/dev/null 2>&1; then
+    echo "Adding user $RUN_UID with name appuser"
+    useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser
 fi
 
 # Get username for the UID (whether we just created it or it existed)
-USERNAME=$(getent passwd "$UID" | cut -d: -f1)
-echo "Username for UID $UID is $USERNAME"
+USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1)
+echo "Username for UID $RUN_UID is $USERNAME"
 
 test_write() {
     folder=$1
-    test_file=$folder/calibre-web-automated-book-downloader_TEST_WRITE
+    test_file=$folder/shelfmark_TEST_WRITE
     mkdir -p $folder
     (
         echo 0123456789_TEST | sudo -E -u "$USERNAME" HOME=/app tee $test_file > /dev/null
@@ -84,7 +157,16 @@ make_writable() {
     else
         echo "Folder $folder is not writable, changing ownership"
         change_ownership $folder
-        chmod g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        chmod -R g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+    fi
+    # Fix any misowned subdirectories/files (e.g., from previous runs as root)
+    if [ -d "$folder" ]; then
+        misowned_count=$(find "$folder" -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) 2>/dev/null | wc -l)
+        if [ "$misowned_count" -gt 0 ]; then
+            echo "Fixing ownership of $misowned_count files/directories in $folder"
+            find "$folder" -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+                -exec chown "$RUN_UID:$RUN_GID" {} \; 2>/dev/null || true
+        fi
     fi
     test_write $folder || echo "Failed to test write to ${folder}, continuing..."
 }
@@ -93,23 +175,88 @@ make_writable() {
 change_ownership() {
   folder=$1
   mkdir -p $folder
-  echo "Changing ownership of $folder to $USERNAME:$GID"
-  chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
-  chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
+  echo "Changing ownership of $folder to $USERNAME:$RUN_GID"
+  chown -R "${RUN_UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
+  chown -R ":${RUN_GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
 }
 
 change_ownership /app
-change_ownership /var/log/cwa-book-downloader
-change_ownership /tmp/cwa-book-downloader
+change_ownership /var/log/shelfmark
+change_ownership /tmp/shelfmark
+
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
+
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
+fi
 
 # Test write to all folders
-make_writable /cwa-book-ingest
+make_writable ${CONFIG_DIR:-/config}
+make_writable ${INGEST_DIR:-/books}
+
+# Fix permissions on directories configured in settings
+echo "Checking for additional configured directories..."
+if [ -f /app/scripts/fix_permissions.py ]; then
+    configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
+    if [ -n "$configured_dirs" ]; then
+        echo "$configured_dirs" | while read -r dir; do
+            if [ -n "$dir" ] && [ -d "$dir" ]; then
+                echo "Checking configured directory: $dir"
+                make_writable "$dir"
+            fi
+        done
+    fi
+fi
+
+# Fallback to root if config dir is still not writable (common on NAS/Unraid after upgrade from v0.4.0)
+CONFIG_PATH=${CONFIG_DIR:-/config}
+set +e
+test_write "$CONFIG_PATH" >/dev/null 2>&1
+config_ok=$?
+set -e
+
+if [ $config_ok -ne 0 ] && [ "$RUN_UID" != "0" ]; then
+    config_owner=$(stat -c '%u' "$CONFIG_PATH" 2>/dev/null || echo "unknown")
+    if [ "$config_owner" = "0" ]; then
+        echo ""
+        echo "========================================================"
+        echo "WARNING: Permission issue detected!"
+        echo ""
+        echo "Config directory is owned by root but PUID=$RUN_UID."
+        echo "This typically happens after upgrading from v0.4.0 where"
+        echo "PUID/PGID settings were not respected."
+        echo ""
+        echo "Falling back to running as root to prevent data loss."
+        echo ""
+        echo "To fix this permanently, run on your HOST machine:"
+        echo "  chown -R $RUN_UID:$RUN_GID /path/to/config"
+        echo ""
+        echo "Then restart the container."
+        echo "========================================================"
+        echo ""
+        RUN_UID=0
+        RUN_GID=0
+        USERNAME=root
+    fi
+fi
 
 # Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
 # upgrades work reliably on customer machines.
 # Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
 gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
-command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} app:app"
+command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
 
 # If DEBUG and not using an external bypass
 if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
@@ -165,18 +312,24 @@ if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
 fi
 
-# Hacky way to verify /tmp has at least 1MB of space and is writable/readable
+# Verify /tmp has at least 1MB of space and is writable/readable
 echo "Verifying /tmp has enough space"
-rm -f /tmp/test.cwa-bd
-for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd
-sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))")
-[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1)
-rm /tmp/test.cwa-bd
+rm -f /tmp/test.shelfmark
+if dd if=/dev/zero of=/tmp/test.shelfmark bs=1M count=1 2>/dev/null && \
+   [ "$(wc -c < /tmp/test.shelfmark)" -eq 1048576 ]; then
+    rm -f /tmp/test.shelfmark
+    echo "Success: /tmp is writable and readable"
+else
+    echo "Failure: /tmp is not writable or has insufficient space"
+    exit 1
+fi
 
 echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
+# Set umask for file permissions (default: 0022 = files 644, dirs 755)
+UMASK_VALUE=${UMASK:-0022}
+echo "Setting umask to $UMASK_VALUE"
+umask $UMASK_VALUE
 
+stop_file_logging
 exec sudo -E -u "$USERNAME" HOME=/app $command

env.py

@@ -1,98 +0,0 @@
-import os
-from pathlib import Path
-
-def string_to_bool(s: str) -> bool:
-    return s.lower() in ["true", "yes", "1", "y"]
-
-# Authentication and session settings
-SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
-
-CWA_DB = os.getenv("CWA_DB_PATH")
-CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
-LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
-LOG_DIR = LOG_ROOT / "cwa-book-downloader"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
-INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
-INGEST_DIR_BOOK_FICTION = os.getenv("INGEST_DIR_BOOK_FICTION", "")
-INGEST_DIR_BOOK_NON_FICTION = os.getenv("INGEST_DIR_BOOK_NON_FICTION", "")
-INGEST_DIR_BOOK_UNKNOWN = os.getenv("INGEST_DIR_BOOK_UNKNOWN", "")
-INGEST_DIR_MAGAZINE = os.getenv("INGEST_DIR_MAGAZINE", "")
-INGEST_DIR_COMIC_BOOK = os.getenv("INGEST_DIR_COMIC_BOOK", "")
-INGEST_DIR_AUDIOBOOK = os.getenv("INGEST_DIR_AUDIOBOOK", "")
-INGEST_DIR_STANDARDS_DOCUMENT = os.getenv("INGEST_DIR_STANDARDS_DOCUMENT", "")
-INGEST_DIR_MUSICAL_SCORE = os.getenv("INGEST_DIR_MUSICAL_SCORE", "")
-INGEST_DIR_OTHER = os.getenv("INGEST_DIR_OTHER", "")
-DOWNLOAD_PATHS = {
-    "book (fiction)": Path(INGEST_DIR_BOOK_FICTION) if INGEST_DIR_BOOK_FICTION else INGEST_DIR,
-    "book (non-fiction)": Path(INGEST_DIR_BOOK_NON_FICTION) if INGEST_DIR_BOOK_NON_FICTION else INGEST_DIR,
-    "book (unknown)": Path(INGEST_DIR_BOOK_UNKNOWN) if INGEST_DIR_BOOK_UNKNOWN else INGEST_DIR,
-    "magazine": Path(INGEST_DIR_MAGAZINE) if INGEST_DIR_MAGAZINE else INGEST_DIR,
-    "comic book": Path(INGEST_DIR_COMIC_BOOK) if INGEST_DIR_COMIC_BOOK else INGEST_DIR,
-    "audiobook": Path(INGEST_DIR_AUDIOBOOK) if INGEST_DIR_AUDIOBOOK else INGEST_DIR,
-    "standards document": Path(INGEST_DIR_STANDARDS_DOCUMENT) if INGEST_DIR_STANDARDS_DOCUMENT else INGEST_DIR,
-    "musical score": Path(INGEST_DIR_MUSICAL_SCORE) if INGEST_DIR_MUSICAL_SCORE else INGEST_DIR,
-    "other": Path(INGEST_DIR_OTHER) if INGEST_DIR_OTHER else INGEST_DIR,
-}
-
-STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
-USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
-MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
-DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
-USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
-HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
-HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
-AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
-_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
-_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
-_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
-_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
-_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
-FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
-FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
-# Debug: skip specific download sources for testing fallback chains
-# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
-_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
-DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
-PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
-ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
-
-# Version information from Docker build
-BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
-RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
-
-# If debug is true, we want to log everything
-if DEBUG:
-    LOG_LEVEL = "DEBUG"
-else:
-    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
-MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
-MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1"))
-DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip()
-USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
-BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
-BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true"))
-
-# Logging settings
-LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
-
-USING_EXTERNAL_BYPASSER = string_to_bool(os.getenv("USING_EXTERNAL_BYPASSER", "false"))
-if USING_EXTERNAL_BYPASSER:
-    EXT_BYPASSER_URL = os.getenv("EXT_BYPASSER_URL", "http://flaresolverr:8191").strip()
-    EXT_BYPASSER_PATH = os.getenv("EXT_BYPASSER_PATH", "/v1").strip()
-    EXT_BYPASSER_TIMEOUT = int(os.getenv("EXT_BYPASSER_TIMEOUT", "60000"))
-
-USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
-# If using Tor, we don't need to set custom DNS, use DOH, or proxy
-if USING_TOR:
-    _CUSTOM_DNS = ""
-    USE_DOH = False
-    HTTP_PROXY = ""
-    HTTPS_PROXY = ""
-
-# Calibre-Web URL for navigation button
-CALIBRE_WEB_URL = os.getenv("CALIBRE_WEB_URL", "").strip()
-    

genDebug.sh

@@ -2,8 +2,8 @@
 
 # Set up log paths
 LOG_ROOT=${LOG_ROOT:-"/var/log"}
-LOG_DIR="$LOG_ROOT/cwa-book-downloader"
-OUTPUT_FILE_NAME="cwa-book-downloader-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
+LOG_DIR="$LOG_ROOT/shelfmark"
+OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
 OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
 
 # Create LOG_DIR if it doesn't exist
@@ -18,17 +18,17 @@ echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add disk usage
 echo "=== Disk Usage ===" >> "$LOG_DIR/system_info.txt"
-df -h >> "$LOG_DIR/system_info.txt"
+df -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add memory info
 echo "=== Memory Info ===" >> "$LOG_DIR/system_info.txt"
-free -h >> "$LOG_DIR/system_info.txt"
+free -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add running processes
 echo "=== Running Processes ===" >> "$LOG_DIR/system_info.txt"
-ps aux >> "$LOG_DIR/system_info.txt"
+ps aux >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add network information using basic commands
@@ -37,17 +37,17 @@ echo "=== Network Information ===" > "$LOG_DIR/network_info.txt"
 # Try to get basic connectivity information
 echo "=== Basic Connectivity ===" >> "$LOG_DIR/network_info.txt"
 echo "Hostname resolution:" >> "$LOG_DIR/network_info.txt"
-cat /etc/hosts 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
+cat /etc/hosts >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 echo "DNS configuration:" >> "$LOG_DIR/network_info.txt"
-cat /etc/resolv.conf 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
+cat /etc/resolv.conf >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try to get interface information from /proc
 echo "=== Network Interfaces (/proc) ===" >> "$LOG_DIR/network_info.txt"
 if [ -f "/proc/net/dev" ]; then
-  cat /proc/net/dev >> "$LOG_DIR/network_info.txt"
+  cat /proc/net/dev >> "$LOG_DIR/network_info.txt" 2>&1
 else
   echo "Not available: /proc/net/dev not found" >> "$LOG_DIR/network_info.txt"
 fi
@@ -55,9 +55,9 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try connectivity tests
 echo "=== Internet Connectivity ===" >> "$LOG_DIR/network_info.txt"
-ping -c 3 1.1.1.1 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
+ping -c 3 1.1.1.1 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
-ping -c 3 one.one.one.one 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
+ping -c 3 one.one.one.one >> "$LOG_DIR/network_info.txt" 2>&1 || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test IPv6 connectivity
@@ -77,7 +77,7 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try IPv6 connectivity test using Cloudflare's IPv6 DNS
 echo "Testing IPv6 connectivity to Cloudflare DNS:" >> "$LOG_DIR/network_info.txt"
-ping6 -c 3 2606:4700:4700::1111 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
+ping6 -c 3 2606:4700:4700::1111 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test SSL connectivity
@@ -92,24 +92,36 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Add installed packages
 echo "=== Installed Python Packages ===" > "$LOG_DIR/packages.txt"
-pip list 2>/dev/null >> "$LOG_DIR/packages.txt" || echo "pip not found" >> "$LOG_DIR/packages.txt"
+pip list >> "$LOG_DIR/packages.txt" 2>&1 || echo "pip not found" >> "$LOG_DIR/packages.txt"
 echo "" >> "$LOG_DIR/packages.txt"
 
 # Check Permissions
 echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
-ls -all /app >> "$LOG_DIR/permissions.txt"
+ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt"
+echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
+ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 
+# Check Iptables (NAT)
+echo "=== IPtables NAT Rules ===" > "$LOG_DIR/iptables_nat.txt"
+iptables -t nat -L -v -n >> "$LOG_DIR/iptables_nat.txt" 2>&1
+
+# Check DNS Resolution details
+echo "=== DNS Resolution Test ===" > "$LOG_DIR/dns_test.txt"
+echo "Resolving google.com:" >> "$LOG_DIR/dns_test.txt"
+nslookup google.com >> "$LOG_DIR/dns_test.txt" 2>&1
+echo "" >> "$LOG_DIR/dns_test.txt"
+echo "Resolving check.torproject.org:" >> "$LOG_DIR/dns_test.txt"
+nslookup check.torproject.org >> "$LOG_DIR/dns_test.txt" 2>&1
+
 
 # Check if running in Docker
 echo "=== Container Info ===" > "$LOG_DIR/container_info.txt"
@@ -122,19 +134,58 @@ else
 fi
 
 # Add environment variables (redacting sensitive info)
-env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt"
+env | grep -v -E "(AA_DONATOR_KEY|HARDCOVER_API_KEY|_KEY=|_SECRET=|_PASSWORD=|_TOKEN=)" | sort > "$LOG_DIR/environment.txt"
+
+# Add configuration files (redacting sensitive values)
+CONFIG_DIR=${CONFIG_DIR:-"/config"}
+if [ -d "$CONFIG_DIR" ]; then
+  mkdir -p "$LOG_DIR/config"
+
+  # Copy and redact main settings file
+  if [ -f "$CONFIG_DIR/settings.json" ]; then
+    # Redact sensitive fields (API keys, passwords, tokens)
+    sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+      "$CONFIG_DIR/settings.json" > "$LOG_DIR/config/settings.json" 2>/dev/null
+  fi
+
+  # Copy and redact plugin config files
+  if [ -d "$CONFIG_DIR/plugins" ]; then
+    mkdir -p "$LOG_DIR/config/plugins"
+    for config_file in "$CONFIG_DIR/plugins"/*.json; do
+      if [ -f "$config_file" ]; then
+        filename=$(basename "$config_file")
+        sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+          "$config_file" > "$LOG_DIR/config/plugins/$filename" 2>/dev/null
+      fi
+    done
+  fi
+
+  echo "Configuration files copied (sensitive values redacted)" >> "$LOG_DIR/container_info.txt"
+else
+  echo "Config directory not found at $CONFIG_DIR" >> "$LOG_DIR/container_info.txt"
+fi
 
 echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
-curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt
+curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt 2>&1
 echo "" >> $LOG_DIR/network_info.txt
 echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
-curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt
+curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt 2>&1
 echo "" >> $LOG_DIR/network_info.txt
 echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
-curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt
+curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt 2>&1
 echo "" >> $LOG_DIR/network_info.txt
 echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
-curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt
+curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt 2>&1
+
+# Copy Tor logs if they exist
+if [ -f "/var/log/tor/notices.log" ]; then
+  cp "/var/log/tor/notices.log" "$LOG_DIR/tor_notices.log"
+fi
+
+# Copy Supervisor logs if they exist
+if [ -d "/var/log/supervisor" ]; then
+  cp -rf "/var/log/supervisor/" "$LOG_DIR/supervisor/"
+fi
 
 # Create the zip file directly from LOG_DIR
 ln -s "$LOG_DIR" /tmp/$OUTPUT_FILE_NAME

models.py

@@ -1,432 +0,0 @@
-"""Data structures and models used across the application."""
-
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Tuple
-from enum import Enum
-from datetime import datetime, timedelta
-from threading import Lock, Event
-from pathlib import Path
-import queue
-import re
-import time
-from env import INGEST_DIR, STATUS_TIMEOUT
-
-class QueueStatus(str, Enum):
-    """Enum for possible book queue statuses."""
-    QUEUED = "queued"
-    RESOLVING = "resolving"
-    DOWNLOADING = "downloading"
-    COMPLETE = "complete"
-    AVAILABLE = "available"
-    ERROR = "error"
-    DONE = "done"
-    CANCELLED = "cancelled"
-
-@dataclass
-class QueueItem:
-    """Queue item with priority and metadata."""
-    book_id: str
-    priority: int
-    added_time: float
-    
-    def __lt__(self, other):
-        """Compare items for priority queue (lower priority number = higher precedence)."""
-        if self.priority != other.priority:
-            return self.priority < other.priority
-        return self.added_time < other.added_time
-
-@dataclass
-class BookInfo:
-    """Data class representing book information."""
-    id: str
-    title: str
-    preview: Optional[str] = None
-    author: Optional[str] = None
-    publisher: Optional[str] = None
-    year: Optional[str] = None
-    language: Optional[str] = None
-    content: Optional[str] = None
-    format: Optional[str] = None
-    size: Optional[str] = None
-    info: Optional[Dict[str, List[str]]] = None
-    description: Optional[str] = None
-    download_urls: List[str] = field(default_factory=list)
-    download_path: Optional[str] = None
-    priority: int = 0
-    progress: Optional[float] = None
-    status_message: Optional[str] = None  # Detailed status message for UI display
-    added_time: Optional[float] = None  # Timestamp when added to queue
-
-    def get_filename(self, fallback_url: Optional[str] = None) -> str:
-        """Build sanitized filename: 'Author - Title (Year).format'
-
-        Resolves format from self.format, download_urls, or fallback_url.
-
-        Args:
-            fallback_url: URL to extract format from if not already known
-
-        Returns:
-            Sanitized filename safe for filesystem use
-        """
-        # Resolve format if needed
-        if not self.format:
-            for url in (self.download_urls[0] if self.download_urls else None, fallback_url):
-                if url:
-                    ext = url.split(".")[-1].lower()
-                    if ext and len(ext) <= 5 and ext.isalnum():
-                        self.format = ext
-                        break
-
-        # Build filename
-        parts = []
-        if self.author:
-            parts.append(self.author)
-            parts.append(" - ")
-        parts.append(self.title)
-        if self.year:
-            parts.append(f" ({self.year})")
-
-        filename = "".join(parts)
-        filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
-
-        if self.format:
-            filename = f"{filename}.{self.format}"
-
-        return filename
-
-class BookQueue:
-    """Thread-safe book queue manager with priority support and cancellation."""
-    def __init__(self) -> None:
-        self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
-        self._lock = Lock()
-        self._status: dict[str, QueueStatus] = {}
-        self._book_data: dict[str, BookInfo] = {}
-        self._status_timestamps: dict[str, datetime] = {}  # Track when each status was last updated
-        self._status_timeout = timedelta(seconds=STATUS_TIMEOUT)  # 1 hour timeout
-        self._cancel_flags: dict[str, Event] = {}  # Cancellation flags for active downloads
-        self._active_downloads: dict[str, bool] = {}  # Track currently downloading books
-    
-    def add(self, book_id: str, book_data: BookInfo, priority: int = 0) -> None:
-        """Add a book to the queue with specified priority.
-        
-        Args:
-            book_id: Unique identifier for the book
-            book_data: Book information
-            priority: Priority level (lower number = higher priority)
-        """
-        with self._lock:
-            # Don't add if already exists and not in error/done state
-            if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                return
-
-            added_time = time.time()
-            book_data.priority = priority
-            book_data.added_time = added_time
-            queue_item = QueueItem(book_id, priority, added_time)
-            self._queue.put(queue_item)
-            self._book_data[book_id] = book_data
-            self._update_status(book_id, QueueStatus.QUEUED)
-    
-    def get_next(self) -> Optional[Tuple[str, Event]]:
-        """Get next book ID from queue with cancellation flag.
-
-        Returns:
-            Tuple of (book_id, cancel_flag) or None if queue is empty
-        """
-        # Use iterative approach to avoid stack overflow if many items are cancelled
-        while True:
-            try:
-                queue_item = self._queue.get_nowait()
-                book_id = queue_item.book_id
-
-                with self._lock:
-                    # Check if book was cancelled while in queue
-                    if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED:
-                        continue  # Skip cancelled items, try next
-
-                    # Create cancellation flag for this download
-                    cancel_flag = Event()
-                    self._cancel_flags[book_id] = cancel_flag
-                    self._active_downloads[book_id] = True
-
-                return book_id, cancel_flag
-            except queue.Empty:
-                return None
-            
-    def _update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Internal method to update status and timestamp."""
-        self._status[book_id] = status
-        self._status_timestamps[book_id] = datetime.now()
-            
-    def update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Update status of a book in the queue."""
-        with self._lock:
-            self._update_status(book_id, status)
-            
-            # Clean up active download tracking when finished
-            if status in [QueueStatus.COMPLETE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                self._active_downloads.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-    
-    def update_download_path(self, book_id: str, download_path: str) -> None:
-        """Update the download path of a book in the queue."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].download_path = download_path
-                
-    def update_progress(self, book_id: str, progress: float) -> None:
-        """Update download progress for a book."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].progress = progress
-    
-    def update_status_message(self, book_id: str, message: str) -> None:
-        """Update detailed status message for a book."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].status_message = message
-            
-    def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]:
-        """Get current queue status."""
-        self.refresh()
-        with self._lock:
-            result: Dict[QueueStatus, Dict[str, BookInfo]] = {status: {} for status in QueueStatus}
-            for book_id, status in self._status.items():
-                if book_id in self._book_data:
-                    result[status][book_id] = self._book_data[book_id]
-            return result
-            
-    def get_queue_order(self) -> List[Dict[str, any]]:
-        """Get current queue order for display."""
-        with self._lock:
-            queue_items = []
-            
-            # Get items from priority queue without removing them
-            temp_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    temp_items.append(item)
-                    if item.book_id in self._book_data:
-                        book_info = self._book_data[item.book_id]
-                        queue_items.append({
-                            'id': item.book_id,
-                            'title': book_info.title,
-                            'author': book_info.author,
-                            'priority': item.priority,
-                            'added_time': item.added_time,
-                            'status': self._status.get(item.book_id, QueueStatus.QUEUED)
-                        })
-                except queue.Empty:
-                    break
-            
-            # Put items back in queue
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
-            
-    def cancel_download(self, book_id: str) -> bool:
-        """Cancel a download or clear a completed/errored item.
-
-        Args:
-            book_id: Book identifier to cancel or clear
-
-        Returns:
-            bool: True if cancellation/clearing was successful
-        """
-        with self._lock:
-            current_status = self._status.get(book_id)
-
-            # Allow cancellation during any active state
-            if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]:
-                # Signal active download to stop
-                if book_id in self._cancel_flags:
-                    self._cancel_flags[book_id].set()
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            elif current_status == QueueStatus.QUEUED:
-                # Remove from queue and mark as cancelled
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
-                # Clear completed/errored/cancelled items from tracking
-                self._status.pop(book_id, None)
-                self._status_timestamps.pop(book_id, None)
-                self._book_data.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-                self._active_downloads.pop(book_id, None)
-                return True
-
-            return False
-            
-    def set_priority(self, book_id: str, new_priority: int) -> bool:
-        """Change the priority of a queued book.
-        
-        Args:
-            book_id: Book identifier
-            new_priority: New priority level (lower = higher priority)
-            
-        Returns:
-            bool: True if priority was successfully changed
-        """
-        with self._lock:
-            if book_id not in self._status or self._status[book_id] != QueueStatus.QUEUED:
-                return False
-                
-            # Remove book from queue and re-add with new priority
-            temp_items = []
-            found = False
-            
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    if item.book_id == book_id:
-                        # Create new item with updated priority
-                        new_item = QueueItem(book_id, new_priority, item.added_time)
-                        temp_items.append(new_item)
-                        found = True
-                        # Update book data priority
-                        if book_id in self._book_data:
-                            self._book_data[book_id].priority = new_priority
-                    else:
-                        temp_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return found
-            
-    def reorder_queue(self, book_priorities: Dict[str, int]) -> bool:
-        """Bulk reorder queue by setting new priorities.
-        
-        Args:
-            book_priorities: Dict mapping book_id to new priority
-            
-        Returns:
-            bool: True if reordering was successful
-        """
-        with self._lock:
-            # Extract all items from queue
-            all_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    # Update priority if specified
-                    if item.book_id in book_priorities:
-                        new_priority = book_priorities[item.book_id]
-                        item = QueueItem(item.book_id, new_priority, item.added_time)
-                        # Update book data priority
-                        if item.book_id in self._book_data:
-                            self._book_data[item.book_id].priority = new_priority
-                    all_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back with updated priorities
-            for item in all_items:
-                self._queue.put(item)
-                
-            return True
-            
-    def get_active_downloads(self) -> List[str]:
-        """Get list of currently active download book IDs."""
-        with self._lock:
-            return list(self._active_downloads.keys())
-    
-    def has_pending_work(self) -> bool:
-        """Check if there are any active downloads or queued items.
-        
-        This is useful for determining if the bypasser should stay active
-        even when the UI is closed.
-        
-        Returns:
-            bool: True if there are active downloads or queued items
-        """
-        with self._lock:
-            # Check for active downloads
-            if self._active_downloads:
-                return True
-            
-            # Check for queued items (excluding cancelled ones)
-            for book_id, status in self._status.items():
-                if status == QueueStatus.QUEUED:
-                    return True
-            
-            return False
-            
-    def clear_completed(self) -> int:
-        """Remove all completed, errored, or cancelled books from tracking.
-        
-        Returns:
-            int: Number of books removed
-        """
-        with self._lock:
-            to_remove = []
-            for book_id, status in self._status.items():
-                if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
-                    to_remove.append(book_id)
-            
-            removed_count = len(to_remove)
-            for book_id in to_remove:
-                self._status.pop(book_id, None)
-                self._status_timestamps.pop(book_id, None)
-                self._book_data.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-                self._active_downloads.pop(book_id, None)
-                
-            return removed_count
-        
-    def refresh(self) -> None:
-        """Remove any books that are done downloading or have stale status."""
-        with self._lock:
-            current_time = datetime.now()
-            
-            # Create a list of items to remove to avoid modifying dict during iteration
-            to_remove = []
-            
-            for book_id, status in self._status.items():
-                path = self._book_data[book_id].download_path
-                if path and not Path(path).exists():
-                    self._book_data[book_id].download_path = None
-                    path = None
-                
-                # Check for completed downloads
-                if status == QueueStatus.AVAILABLE:
-                    if not path:
-                        self._update_status(book_id, QueueStatus.DONE)
-                
-                # Check for stale status entries
-                last_update = self._status_timestamps.get(book_id)
-                if last_update and (current_time - last_update) > self._status_timeout:
-                    if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]:
-                        to_remove.append(book_id)
-            
-            # Remove stale entries
-            for book_id in to_remove:
-                del self._status[book_id]
-                del self._status_timestamps[book_id]
-                if book_id in self._book_data:
-                    del self._book_data[book_id]
-
-    def set_status_timeout(self, hours: int) -> None:
-        """Set the status timeout duration in hours."""
-        with self._lock:
-            self._status_timeout = timedelta(hours=hours)
-
-
-# Global instance of BookQueue
-book_queue = BookQueue()
-
-@dataclass
-class SearchFilters:
-    isbn: Optional[List[str]] = None
-    author: Optional[List[str]] = None
-    title: Optional[List[str]] = None
-    lang: Optional[List[str]] = None
-    sort: Optional[str] = None
-    content: Optional[List[str]] = None
-    format: Optional[List[str]] = None

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "shelfmark"
+version = "0.1.0"
+description = "Shelfmark - Book Downloader"
+requires-python = ">=3.10"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--tb=short",
+]
+markers = [
+    "integration: marks tests that require running services (deselect with '-m \"not integration\"')",
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks end-to-end tests that require the full application stack",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+ignore_missing_imports = true

readme.md

@@ -1,365 +1,258 @@
-# 📚 Calibre-Web-Automated-Book-Downloader
+# 📚 Shelfmark: Book Downloader
 
-<img src="src/frontend/public/logo.png" alt="Calibre-Web Automated Book Downloader" width="200">
+Formerly *Calibre Web Automated Book Downloader (CWABD)*
 
-An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
+<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
+
+Shelfmark is a unified web interface for searching and aggregating books and audiobook downloads from multiple sources - all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library.
+
+**Fully standalone** - no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated), [Booklore](https://github.com/booklore-app/booklore) or [Audiobookshelf](https://github.com/advplyr/audiobookshelf) for automatic import.
 
 ## ✨ Features
 
-- 🌐 User-friendly web interface for book search and download
-- 🔄 Automated download to your specified ingest folder
-- 🔌 Seamless integration with Calibre-Web-Automated
-- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr)
-- 🛡️ Cloudflare bypass capability for reliable downloads
-- 🐳 Docker-based deployment for quick setup
+- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
+- **Multiple sources** - Popular archive websites, Torrent, Usenet and IRC download support
+- **Audiobook support** - Full audiobook search and download with dedicated processing
+- **Real-Time Progress** - Unified download queue with live status updates across all sources
+- **Two Search Modes**:
+  - **Direct** - Search popular web sources
+  - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
+- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
 
 ## 🖼️ Screenshots
 
-![Main search interface Screenshot](README_images/search.png 'Main search interface')
+**Home screen**
+![Home screen](README_images/homescreen.png 'Home screen')
 
-![Details modal Screenshot placeholder](README_images/details.png 'Details modal')
+**Search results**
+![Search results](README_images/search-results.png 'Search results')
 
-![Download queue Screenshot placeholder](README_images/downloading.png 'Download queue')
+**Multi-source downloads**
+![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
+
+**Download queue**
+![Download queue](README_images/downloads.png 'Download queue')
 
 ## 🚀 Quick Start
 
 ### Prerequisites
 
-- Docker
-- Docker Compose
-- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended)
+- Docker & Docker Compose
 
-### Installation Steps
-
-1. Get the docker-compose.yml:
+### Installation
 
+1. Download the [docker-compose file](compose/docker-compose.yml):
    ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml
+   curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
    ```
 
 2. Start the service:
-
    ```bash
    docker compose up -d
    ```
 
-3. Access the web interface at `http://localhost:8084`
+3. Open `http://localhost:8084`
 
-## ⚙️ Configuration
+That's it! Configure settings through the web interface as needed.
 
-### Environment Variables
-
-#### Application Settings
-
-| Variable          | Description             | Default Value      |
-| ----------------- | ----------------------- | ------------------ |
-| `FLASK_PORT`      | Web interface port      | `8084`             |
-| `FLASK_HOST`      | Web interface binding   | `0.0.0.0`          |
-| `DEBUG`           | Debug mode toggle       | `false`            |
-| `INGEST_DIR`      | Book download directory | `/cwa-book-ingest` |
-| `TZ`              | Container timezone      | `UTC`              |
-| `UID`             | Runtime user ID         | `1000`             |
-| `GID`             | Runtime group ID        | `100`              |
-| `CWA_DB_PATH`     | Calibre-Web's database  | None               |
-| `ENABLE_LOGGING`  | Enable log file         | `true`             |
-| `LOG_LEVEL`       | Log level to use        | `info`             |
-| `SESSION_COOKIE_SECURE` | Secure cookie enforcement - Use for HTTPS connections only | `false` |
-| `CALIBRE_WEB_URL` | Custom WebUI library link | None |
-| `BYPASS_WARMUP_ON_CONNECT` | Warm up Cloudflare bypasser when first client connects | `true` |
-
-If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
-
-Set `CALIBRE_WEB_URL` to your Calibre-Web / Booklore base URL. A ‘Go to library’ button will appear in the Web UI for quick access while downloading, and it also provides library access when CWA-BD is installed as a mobile PWA.
-
-If logging is enabled, log folder default location is `/var/log/cwa-book-downloader`
-Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages.
-
-Note that if using TOR, the TZ will be calculated automatically based on IP.
-
-#### Download Settings
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `MAX_RETRY`            | Maximum retry attempts                                    | `3`                               |
-| `DEFAULT_SLEEP`        | Retry delay (seconds)                                     | `5`                               |
-| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds)                           | `5`                               |
-| `SUPPORTED_FORMATS`    | Supported book formats                                    | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
-| `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
-| `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
-| `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
-| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
-| `ALLOW_USE_WELIB`       | Allow usage of welib for downloading books if found there | `true`                            |
-
-If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
-
-Use the following environment variables to set specific folders in which to download 
-different content types (Book, Magazine, Comic, etc.):
-
-| Variable                        | Description                    | Default Value |
-|---------------------------------|--------------------------------|---------------|
-| `INGEST_DIR_BOOK_FICTION`       | Book (fiction) folder name     | ``            |
-| `INGEST_DIR_BOOK_NON_FICTION`   | Book (non-fiction) folder name | ``            |
-| `INGEST_DIR_BOOK_UNKNOWN`       | Book (unknown) folder name     | ``            |
-| `INGEST_DIR_MAGAZINE`           | Magazine folder name           | ``            |
-| `INGEST_DIR_COMIC_BOOK`         | Comic book folder name         | ``            |
-| `INGEST_DIR_AUDIOBOOK`          | Audiobook folder name          | ``            |
-| `INGEST_DIR_STANDARDS_DOCUMENT` | Standards document folder name | ``            |
-| `INGEST_DIR_MUSICAL_SCORE`      | Musical score folder name      | ``            |
-
-If no specific path is set for a content type the default is `INGEST_DIR`.  
-Remember to map the specified paths to where your instance of Calibre-Web-Automated (CWA) will find them, e.g.:  
-```
-volumes:
-    - /tmp/data/calibre-web/comicbook-ingest:/cwa-comicbook-ingest
-```
-if `INGEST_DIR_COMIC_BOOK=/cwa-comicbook-ingest` and your CWA is configured to use `/tmp/data/calibre-web/comicbook-ingest` 
-for comic books.
-
-
-#### AA 
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
-
-If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
-If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.
-
-#### Network Settings
-
-| Variable               | Description                     | Default Value           |
-| ---------------------- | ------------------------------- | ----------------------- |
-| `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
-| `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
-| `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | DNS configuration               | `auto`                  |
-| `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
-
-**Proxy Configuration**
-
-For proxy configuration, you can specify URLs in the following format:
-```bash
-# Basic proxy
-HTTP_PROXY=http://proxy.example.com:8080
-HTTPS_PROXY=http://proxy.example.com:8080
-
-# Proxy with authentication
-HTTP_PROXY=http://username:password@proxy.example.com:8080
-HTTPS_PROXY=http://username:password@proxy.example.com:8080
-```
-
-**DNS Configuration**
-
-The `CUSTOM_DNS` setting controls how DNS resolution works. By default, it is set to `auto` which provides automatic failover for reliable connectivity.
-
-**Auto Mode (Default)**
-
-When `CUSTOM_DNS=auto`, the application starts with your system's default DNS. If DNS resolution fails, it automatically rotates through alternative providers using DNS over HTTPS (DoH):
-
-1. System DNS (initial)
-2. Cloudflare (1.1.1.1)
-3. Google (8.8.8.8)
-4. Quad9 (9.9.9.9)
-5. OpenDNS (208.67.222.222)
-
-This automatic rotation helps bypass ISP-level blocks and DNS issues without any manual configuration.
-
-**Manual DNS Configuration**
-
-If you prefer to use a specific DNS configuration, you can override the auto behavior:
-
-1. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS (8.8.8.8, 8.8.4.4)
-   - `quad9` - Quad9 DNS (9.9.9.9, 149.112.112.112)
-   - `cloudflare` - Cloudflare DNS (1.1.1.1, 1.0.0.1)
-   - `opendns` - OpenDNS (208.67.222.222, 208.67.220.220)
-
-2. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
-   - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses
-
-When using preset providers, you can optionally enable DNS over HTTPS with `USE_DOH=true`:
-```bash
-CUSTOM_DNS=cloudflare
-USE_DOH=true
-```
-
-Note: When using custom IP addresses, the `USE_DOH` flag is ignored since DoH requires a known provider endpoint.
-
-#### Custom configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `CUSTOM_SCRIPT`        | Path to an executable script that tuns after each download  | ``                      |
-
-If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation.
-
-The script is called with the full path of the downloaded file as its argument. Important notes:
-- The script must preserve the original filename for proper processing
-- The file can be modified or even deleted if needed
-- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted)
-
-You can specify these configuration in this format :
-```
-environment:
-  - CUSTOM_SCRIPT=/scripts/process-book.sh
-
-volumes:
-  - local/scripts/custom_script.sh:/scripts/process-book.sh
-```
-
-### Volume Configuration
+### Volume Setup
 
 ```yaml
 volumes:
-  - /your/local/path:/cwa-book-ingest
-  - /cwa/config/path/app.db:/auth/app.db:ro
+  - /your/config/path:/config # Config, database, and artwork cache directory
+  - /your/download/path:/books # Downloaded books
+  - /client/path:/client/path # Optional: For Torrent/Usenet downloads, match your client directory exactly. 
 ```
-**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777
 
-Mount should align with your Calibre-Web-Automated ingest folder.
+> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
 
-## Variants:
+> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
 
-### 🧅 Tor Variant
+## ⚙️ Configuration
 
-This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions.
+### Search Modes
 
-To use the Tor variant:
+**Direct** (default)
+- Works out of the box, no setup required
+- Searches a huge library of books directly
+- Returns downloadable releases immediately
 
-1.  Get the Tor-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.tor.yml up -d
-    ```
+**Universal**
+- Cleaner search results via metadata providers (Hardcover is recommended)
+- Aggregates releases from multiple configured sources
+- Full Audiobook support
+- Requires manual setup (API keys, additional sources)
 
-**Important Considerations for Tor:**
+### Environment Variables
 
-*   **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying.
-*   **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set.
-*   **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor.
+Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
 
-### External Cloudflare resolver variant
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FLASK_PORT` | Web interface port | `8084` |
+| `INGEST_DIR` | Book download directory | `/books` |
+| `TZ` | Container timezone | `UTC` |
+| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` |
+| `SEARCH_MODE` | `direct` or `universal` | `direct` |
+| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
 
-This variant allows the application to use an external service to bypass Cloudflare protection, instead of relying on the built-in bypasser. This is useful if you already have a dedicated Cloudflare resolver (such as [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) or compatible services like [ByParr](https://github.com/ThePhaseless/Byparr)) running elsewhere.
+See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
 
-#### How it works:
+Some of the additional options available in Settings:
+- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
+- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
+- **IRC** - Add details for IRC book sources and download directly from the UI
+- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
+- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
+- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
+- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
+- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
 
-- When enabled, all requests that require Cloudflare bypass are sent to your external resolver service.
-- The application communicates with the resolver using its API.
+## 🐳 Docker Variants
 
-#### Configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `EXT_BYPASSER_URL`     | The full URL of your external resolver (required)           |                         |
-| `EXT_BYPASSER_PATH`    | API path for the resolver (usually `/v1`)                   | `/v1`                   |
-| `EXT_BYPASSER_TIMEOUT` | Timeout for page loading (in milliseconds)                  | `60000`                 |
-
-#### Important
-
-This feature follows the same configuration of the built-in Cloudflare bypasser, so you should turn on the `USE_CF_BYPASS` configuration to enable it.
-
-#### To use the External Cloudflare resolver variant:
-
-1.  Get the extbp-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.extbp.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.extbp.yml up -d
-    ```
-
-#### Compatibility:
-This feature is designed to work with any resolver that implements the `FlareSolverr` API schema, including `ByParr` and similar projects.
-
-#### Internal vs External Bypasser
-
-The **internal bypasser** (default) is custom-designed for this application's specific needs. It handles session management, cookie persistence, and retry logic optimized for book downloading workflows. For most users, this provides the most reliable experience out of the box.
-
-The **external bypasser** is better suited if you:
-- Already run FlareSolverr/ByParr for other services and want to consolidate
-- Need to share bypass infrastructure across multiple applications
-- Want to offload browser automation to a dedicated, more powerful container
-
-If you're unsure which to use, start with the default internal bypasser.
-
-## 🏗️ Architecture
-
-The application consists of a Flask backend with a React-based frontend:
-
-### Backend
-- **Flask Application**: Python-based backend (`app.py`, `backend.py`) providing REST API and WebSocket support
-- **Download Manager**: Handles book search, download requests, and queue management (`downloader.py`, `book_manager.py`)
-- **Network Layer**: Cloudflare bypass and proxy support (`cloudflare_bypasser.py`, `network.py`)
-
-### Frontend
-- **React + TypeScript**: Modern web interface built with Vite (`src/frontend`)
-- **Real-time Updates**: WebSocket integration for live download status
-- **Responsive UI**: TailwindCSS-based design for mobile and desktop
-
-For frontend development, use the provided Makefile:
+### Standard
 ```bash
-make install  # Install dependencies
-make dev      # Start development server
-make build    # Build for production
-```
-If you run the docker compose file, the frontend will be built and served automatically. But if you run the frontend dev server it will supercede the docker compose frontend.
-
-## 🏥 Health Monitoring
-
-Built-in health checks monitor:
-
-- Web interface availability
-- Download service status
-- Cloudflare bypass service connection
-
-Checks run every 30 seconds with a 30-second timeout and 3 retries.
-You can enable by adding this to your compose :
-```
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD curl -s http://localhost:8084/api/status || exit 1
+docker compose up -d
 ```
 
-## 📝 Logging
+The full-featured image with built-in Cloudflare bypass.
 
-Logs are available in:
+#### Enable Tor Routing
+Routes all traffic through Tor for enhanced privacy:
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
+docker compose -f docker-compose.tor.yml up -d
+```
 
-- Container: `/var/logs/cwa-book-downloader.log`
-- Docker logs: Access via `docker logs`
+**Notes:**
+- Requires `NET_ADMIN` and `NET_RAW` capabilities
+- Timezone is auto-detected from Tor exit node
+- Custom DNS/proxy settings are ignored when Tor is active
 
-## 🤝 Contributing
+### Lite
+A smaller image without the built-in Cloudflare bypasser. Ideal for:
 
-Contributions are welcome! Feel free to submit a Pull Request.
+- **External bypassers** - Already running FlareSolverr or ByParr for other services
+- **Fast downloads** - Using fast download sources
+- **Alternative sources only** - Exclusively using Prowlarr, IRC, or other sources
+- **Audiobooks** - Using Shelfmark exclusively for audiobooks
 
-## 📄 License
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
+docker compose -f docker-compose.lite.yml up -d
+```
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
 
-## ⚠️ Important Disclaimers
+## 🔐 Authentication
+
+Authentication is optional but recommended for shared or exposed instances. Three authentication methods are available in Settings:
+
+**1. Single Username/Password**
+
+**2. Proxy (Forward) Authentication**
+
+Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
+
+**3. Calibre-Web Database**
+
+If you're running Calibre-Web, you can reuse its user database by mounting it:
+
+```yaml
+volumes:
+  - /path/to/calibre-web/app.db:/auth/app.db:ro
+```
+
+## Health Monitoring
+
+The application exposes a health endpoint at `/api/health` (no authentication required). Add a health check to your compose:
+
+```yaml
+healthcheck:
+  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/health"]
+  interval: 30s
+  timeout: 30s
+  retries: 3
+```
+
+## Logging
+
+Logs are available via:
+- `docker logs <container-name>`
+- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
+
+Log level is configurable via Settings or `LOG_LEVEL` environment variable.
+
+## Development
+
+```bash
+# Frontend development
+make install     # Install dependencies
+make dev         # Start Vite dev server (localhost:5173)
+make build       # Production build
+make typecheck   # TypeScript checks
+
+# Backend (Docker)
+make up          # Start backend via docker-compose.dev.yml
+make down        # Stop services
+make refresh     # Rebuild and restart
+make restart     # Restart container
+```
+
+The frontend dev server proxies to the backend on port 8084.
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      Web Interface                          │
+│                 (React + TypeScript + Vite)                 │
+├─────────────────────────────────────────────────────────────┤
+│                      Flask Backend                          │
+│                   (REST API + WebSocket)                    │
+├───────────────────┬─────────────────────┬───────────────────┤
+│ Metadata Providers│   Download Queue    │  Cloudflare       │
+│                   │   & Orchestrator    │  Bypass           │
+├───────────────────┼─────────────────────┼───────────────────┤
+│ • Hardcover       │ • Task scheduling   │ • Internal        │
+│ • Open Library    │ • Progress tracking │ • External        │
+│                   │ • Retry logic       │   (FlareSolverr)  │
+├───────────────────┴─────────────────────┴───────────────────┤
+│                     Release Sources                         │
+├─────────────────────────────────────────────────────────────┤
+│ • Direct Download (Web Sources → Mirrors → Fallbacks)       │
+├─────────────────────────────────────────────────────────────┤
+│                     Network Layer                           │
+├─────────────────────────────────────────────────────────────┤
+│ • Auto DNS rotation  • Mirror failover  • Resume support    │
+└─────────────────────────────────────────────────────────────┘
+```
+
+The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
+
+## Contributing
+
+Contributions are welcome! Please file issues or submit pull requests on GitHub.
+
+> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR!
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## ⚠️ Disclaimers
 
 ### Copyright Notice
 
-While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for:
-
+This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
 - Ensuring they have the right to download requested materials
 - Respecting copyright laws and intellectual property rights
 - Using the tool in compliance with their local regulations
 
-### Duplicate Downloads Warning
+### Library Integration
 
-Please note that the current version:
+Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
 
-- Does not check for existing files in the download directory
-- Does not verify if books already exist in your Calibre database
-- Exercise caution when requesting multiple books to avoid duplicates
-
-## 💬 Support
-
-For issues or questions, please file an issue on the GitHub repository.
+## Support
 
+For issues or questions, please [file an issue](https://github.com/calibrain/shelfmark/issues) on GitHub.

requirements-base.txt

@@ -11,3 +11,6 @@ gevent
 gevent-websocket
 psutil
 emoji
+rarfile
+qbittorrent-api
+transmission-rpc

requirements-shelfmark.txt

@@ -1,4 +1,4 @@
 pyvirtualdisplay
 pyautogui
-seleniumbase>=4.41.1
+seleniumbase>=4.45.6
 python-xlib

scripts/fix_permissions.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fix permissions on all configured directories.
+
+This script is called by the entrypoint to ensure all user-configured
+directories have correct ownership. It reads directory paths from:
+- CONFIG_DIR environment variable
+- Config files in CONFIG_DIR/plugins/
+
+Outputs directory paths that need permission fixing (one per line).
+The entrypoint handles the actual chown operations.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def get_directories_from_config() -> set[str]:
+    """Extract all directory paths from config files."""
+    directories = set()
+
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    plugins_dir = config_dir / "plugins"
+
+    if not plugins_dir.exists():
+        return directories
+
+    # Keys that contain directory paths
+    directory_keys = {
+        # Main destinations
+        "DESTINATION",
+        "DESTINATION_AUDIOBOOK",
+        # Content type routing directories
+        "AA_CONTENT_TYPE_DIR_FICTION",
+        "AA_CONTENT_TYPE_DIR_NON_FICTION",
+        "AA_CONTENT_TYPE_DIR_UNKNOWN",
+        "AA_CONTENT_TYPE_DIR_MAGAZINE",
+        "AA_CONTENT_TYPE_DIR_COMIC",
+        "AA_CONTENT_TYPE_DIR_STANDARDS",
+        "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+        "AA_CONTENT_TYPE_DIR_OTHER",
+        # Legacy keys (in case of old configs)
+        "INGEST_DIR",
+        "INGEST_DIR_AUDIOBOOK",
+        "INGEST_DIR_BOOK_FICTION",
+        "INGEST_DIR_BOOK_NON_FICTION",
+        "INGEST_DIR_BOOK_UNKNOWN",
+        "INGEST_DIR_MAGAZINE",
+        "INGEST_DIR_COMIC_BOOK",
+        "INGEST_DIR_STANDARDS_DOCUMENT",
+        "INGEST_DIR_MUSICAL_SCORE",
+        "INGEST_DIR_OTHER",
+        "LIBRARY_PATH",
+        "LIBRARY_PATH_AUDIOBOOK",
+    }
+
+    # Read all JSON config files
+    for config_file in plugins_dir.glob("*.json"):
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+
+            for key in directory_keys:
+                if key in config:
+                    value = config[key]
+                    if value and isinstance(value, str) and value.startswith("/"):
+                        directories.add(value)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    return directories
+
+
+def main():
+    """Output all configured directories that exist."""
+    directories = get_directories_from_config()
+
+    # Filter to directories that actually exist
+    existing = []
+    for dir_path in directories:
+        path = Path(dir_path)
+        if path.exists() and path.is_dir():
+            existing.append(dir_path)
+
+    # Output one directory per line
+    for dir_path in sorted(existing):
+        print(dir_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_env_docs.py

@@ -0,0 +1,433 @@
+#!/usr/bin/env python3
+"""Generate markdown documentation for environment variables from the settings registry.
+
+This script extracts all settings that support environment variable configuration
+and generates a comprehensive markdown file documenting each option.
+
+Usage:
+    python scripts/generate_env_docs.py [--output path/to/output.md]
+
+The generated documentation includes:
+- Environment variable name
+- Description
+- Type (string, number, boolean, etc.)
+- Default value
+- Organizational grouping by settings tab/group
+"""
+
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Add project root to path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+
+def get_field_type_name(field) -> str:
+    """Get a human-readable type name for a field."""
+    from shelfmark.core.settings_registry import (
+        CheckboxField,
+        MultiSelectField,
+        NumberField,
+        OrderableListField,
+        PasswordField,
+        SelectField,
+        TextField,
+    )
+
+    if isinstance(field, CheckboxField):
+        return "boolean"
+    elif isinstance(field, NumberField):
+        return "number"
+    elif isinstance(field, SelectField):
+        return "string (choice)"
+    elif isinstance(field, MultiSelectField):
+        return "string (comma-separated)"
+    elif isinstance(field, OrderableListField):
+        return "JSON array"
+    elif isinstance(field, PasswordField):
+        return "string (secret)"
+    elif isinstance(field, TextField):
+        return "string"
+    else:
+        return "string"
+
+
+def format_default_value(field) -> str:
+    """Format the default value for display."""
+    default = field.default
+
+    if default is None:
+        return "_none_"
+    elif isinstance(default, bool):
+        return f"`{str(default).lower()}`"
+    elif isinstance(default, (int, float)):
+        return f"`{default}`"
+    elif isinstance(default, str):
+        if default == "":
+            return "_empty string_"
+        return f"`{default}`"
+    elif isinstance(default, list):
+        if not default:
+            return "_empty list_"
+        # For simple lists, show comma-separated values
+        if all(isinstance(item, str) for item in default):
+            return f"`{','.join(default)}`"
+        # For complex lists (e.g., OrderableListField defaults), summarize
+        return f"_see UI for defaults_"
+    else:
+        return f"`{default}`"
+
+
+def get_select_options(field) -> Optional[List[str]]:
+    """Get the available options for a SelectField.
+
+    Returns options formatted as 'value (label)' or just 'value' if they match,
+    so users know the actual values to use in environment variables.
+    """
+    from shelfmark.core.settings_registry import SelectField
+
+    if not isinstance(field, SelectField):
+        return None
+
+    options = field.options
+    if callable(options):
+        try:
+            options = options()
+        except Exception:
+            return None
+
+    if not options:
+        return None
+
+    result = []
+    for opt in options:
+        value = opt.get("value", "")
+        label = opt.get("label", "")
+
+        # Format as "value (label)" unless they're the same or value is empty
+        if value == "":
+            result.append(f'`""` ({label})')
+        elif value == label or not label:
+            result.append(f"`{value}`")
+        else:
+            result.append(f"`{value}` ({label})")
+
+    return result
+
+
+def _generate_bootstrap_env_docs() -> List[str]:
+    """Generate documentation for bootstrap environment variables from env.py."""
+    # These are environment variables defined in env.py that are used before
+    # the settings registry is available
+    bootstrap_vars = [
+        {
+            "name": "CONFIG_DIR",
+            "description": "Directory for storing configuration files and plugin settings.",
+            "type": "string (path)",
+            "default": "/config",
+        },
+        {
+            "name": "LOG_ROOT",
+            "description": "Root directory for log files.",
+            "type": "string (path)",
+            "default": "/var/log/",
+        },
+        {
+            "name": "TMP_DIR",
+            "description": "Staging directory for downloads before moving to destination.",
+            "type": "string (path)",
+            "default": "/tmp/shelfmark",
+        },
+        {
+            "name": "ENABLE_LOGGING",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
+            "type": "boolean",
+            "default": "true",
+        },
+        {
+            "name": "FLASK_HOST",
+            "description": "Host address for the Flask web server.",
+            "type": "string",
+            "default": "0.0.0.0",
+        },
+        {
+            "name": "FLASK_PORT",
+            "description": "Port number for the Flask web server.",
+            "type": "number",
+            "default": "8084",
+        },
+        {
+            "name": "SESSION_COOKIE_SECURE",
+            "description": "Enable secure cookies (requires HTTPS).",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "CWA_DB_PATH",
+            "description": "Path to the Calibre-Web database for authentication integration.",
+            "type": "string (path)",
+            "default": "/auth/app.db",
+        },
+        {
+            "name": "DOCKERMODE",
+            "description": "Indicates the application is running inside a Docker container.",
+            "type": "boolean",
+            "default": "false",
+        },
+    ]
+
+    lines = [
+        "## Bootstrap Configuration",
+        "",
+        "These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
+        "",
+        "| Variable | Description | Type | Default |",
+        "|----------|-------------|------|---------|",
+    ]
+
+    for var in bootstrap_vars:
+        lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
+
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for var in bootstrap_vars:
+        lines.append(f"#### `{var['name']}`")
+        lines.append("")
+        lines.append(var["description"])
+        lines.append("")
+        lines.append(f"- **Type:** {var['type']}")
+        lines.append(f"- **Default:** `{var['default']}`")
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def generate_env_docs() -> str:
+    """Generate markdown documentation for all environment variables."""
+    # Import settings modules to ensure all settings are registered
+    import shelfmark.config.settings  # noqa: F401
+    import shelfmark.release_sources.irc.settings  # noqa: F401
+    import shelfmark.release_sources.prowlarr.settings  # noqa: F401
+    import shelfmark.metadata_providers.hardcover  # noqa: F401
+    import shelfmark.metadata_providers.openlibrary  # noqa: F401
+    import shelfmark.metadata_providers.googlebooks  # noqa: F401
+
+    from shelfmark.core.settings_registry import (
+        ActionButton,
+        HeadingField,
+        get_all_groups,
+        get_all_settings_tabs,
+    )
+
+    tabs = get_all_settings_tabs()
+    groups = {g.name: g for g in get_all_groups()}
+
+    # Organize tabs by group
+    grouped_tabs: Dict[Optional[str], List] = {None: []}
+    for group_name in groups:
+        grouped_tabs[group_name] = []
+
+    for tab in tabs:
+        group_name = tab.group
+        if group_name not in grouped_tabs:
+            grouped_tabs[group_name] = []
+        grouped_tabs[group_name].append(tab)
+
+    # Build markdown output
+    lines = [
+        "# Environment Variables",
+        "",
+        "This document lists all configuration options that can be set via environment variables.",
+        "",
+        "> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
+        "",
+        "## Table of Contents",
+        "",
+    ]
+
+    # Generate TOC
+    toc_entries = [
+        "- [Bootstrap Configuration](#bootstrap-configuration)",
+    ]
+
+    # Ungrouped tabs first
+    for tab in grouped_tabs.get(None, []):
+        anchor = tab.display_name.lower().replace(" ", "-")
+        toc_entries.append(f"- [{tab.display_name}](#{anchor})")
+
+    # Then grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if group_tabs:
+            anchor = group.display_name.lower().replace(" ", "-")
+            toc_entries.append(f"- [{group.display_name}](#{anchor})")
+            for tab in group_tabs:
+                sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
+                toc_entries.append(f"  - [{tab.display_name}](#{sub_anchor})")
+
+    lines.extend(toc_entries)
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Add bootstrap environment variables documentation
+    lines.extend(_generate_bootstrap_env_docs())
+
+    # Generate documentation for ungrouped tabs
+    for tab in grouped_tabs.get(None, []):
+        lines.extend(_generate_tab_docs(tab))
+
+    # Generate documentation for grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if not group_tabs:
+            continue
+
+        lines.append(f"## {group.display_name}")
+        lines.append("")
+
+        for tab in group_tabs:
+            lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
+
+    return "\n".join(lines)
+
+
+def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
+    """Generate documentation for a single settings tab."""
+    from shelfmark.core.settings_registry import ActionButton, HeadingField
+
+    lines = []
+
+    # Section header
+    if group_prefix:
+        lines.append(f"### {group_prefix}: {tab.display_name}")
+        anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
+    else:
+        lines.append(f"## {tab.display_name}")
+
+    lines.append("")
+
+    # Collect env-supported fields
+    env_fields = []
+    for field in tab.fields:
+        # Skip non-value fields
+        if isinstance(field, (ActionButton, HeadingField)):
+            continue
+
+        # Skip fields that don't support ENV vars
+        if not getattr(field, "env_supported", True):
+            continue
+
+        env_fields.append(field)
+
+    if not env_fields:
+        lines.append("_No environment variables for this section._")
+        lines.append("")
+        return lines
+
+    # Generate table
+    lines.append("| Variable | Description | Type | Default |")
+    lines.append("|----------|-------------|------|---------|")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        description = field.description or field.label
+        # Clean up description for table (remove newlines, escape pipes)
+        description = description.replace("\n", " ").replace("|", "\\|").strip()
+
+        field_type = get_field_type_name(field)
+        default = format_default_value(field)
+
+        lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
+
+    lines.append("")
+
+    # Add detailed documentation for each field
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        lines.append(f"#### `{env_var}`")
+        lines.append("")
+        lines.append(f"**{field.label}**")
+        lines.append("")
+
+        if field.description:
+            lines.append(field.description)
+            lines.append("")
+
+        lines.append(f"- **Type:** {get_field_type_name(field)}")
+        lines.append(f"- **Default:** {format_default_value(field)}")
+
+        if getattr(field, "required", False):
+            lines.append("- **Required:** Yes")
+
+        if getattr(field, "requires_restart", False):
+            lines.append("- **Requires restart:** Yes")
+
+        # Show options for SelectField
+        options = get_select_options(field)
+        if options:
+            lines.append(f"- **Options:** {', '.join(options)}")
+
+        # Show constraints for NumberField
+        from shelfmark.core.settings_registry import NumberField
+        if isinstance(field, NumberField):
+            constraints = []
+            if field.min_value is not None:
+                constraints.append(f"min: {field.min_value}")
+            if field.max_value is not None:
+                constraints.append(f"max: {field.max_value}")
+            if constraints:
+                lines.append(f"- **Constraints:** {', '.join(constraints)}")
+
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate markdown documentation for environment variables"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=project_root / "docs" / "environment-variables.md",
+        help="Output file path (default: docs/environment-variables.md)",
+    )
+    parser.add_argument(
+        "--stdout",
+        action="store_true",
+        help="Print to stdout instead of file",
+    )
+    args = parser.parse_args()
+
+    docs = generate_env_docs()
+
+    if args.stdout:
+        print(docs)
+    else:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(docs)
+        print(f"Generated: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_clients.py

@@ -0,0 +1,537 @@
+#!/usr/bin/env python3
+"""
+Test script for download client implementations.
+
+Usage:
+    1. Start the test stack:
+       docker compose -f docker-compose.test-clients.yml up -d
+
+    2. Wait for containers to initialize (first run takes ~30s)
+
+    3. Run this script to verify clients are accessible:
+       python scripts/test_clients.py
+
+    4. Access cwabd at http://localhost:8084
+       - Go to Settings > Prowlarr > Download Clients
+       - Select a client from the dropdown
+       - Click "Test Connection" to verify
+
+Web UIs:
+    - cwabd:        http://localhost:8084
+    - qBittorrent:  http://localhost:8080
+    - Transmission: http://localhost:9091
+    - Deluge:       http://localhost:8112
+    - NZBGet:       http://localhost:6789
+    - SABnzbd:      http://localhost:8085
+    - rTorrent:     http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
+
+Prerequisites (for running this script locally):
+    pip install requests transmission-rpc qbittorrent-api
+
+First-Time Setup:
+    qBittorrent:
+        - Check container logs for temporary password: docker logs test-qbittorrent
+        - Login at http://localhost:8080, change password to something known
+        - Default username is 'admin'
+
+    Transmission:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    Deluge:
+        - Access Web UI at http://localhost:8112 (default password: deluge)
+
+    NZBGet:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    SABnzbd:
+        - Complete the setup wizard at http://localhost:8085
+        - API key will be auto-detected by this script
+        - In cwabd, copy API key from SABnzbd Config > General
+"""
+
+import sys
+import time
+from xmlrpc import client
+
+# Test configuration - matches docker-compose.test-clients.yml
+CONFIG = {
+    # Usenet clients
+    "nzbget": {
+        "url": "http://localhost:6789",
+        "username": "admin",
+        "password": "admin",
+    },
+    "sabnzbd": {
+        "url": "http://localhost:8085",
+        "api_key": None,  # Will be read from config on first run
+    },
+    # Torrent clients
+    "qbittorrent": {
+        "url": "http://localhost:8080",
+        "username": "admin",
+        "password": "5NCngsHXm",  # Temp password from: docker logs test-qbittorrent | grep password
+    },
+    "transmission": {
+        "url": "http://localhost:9091",
+        "username": "admin",
+        "password": "admin",
+    },
+    "deluge": {
+        "url": "http://localhost:8112",
+        "password": "deluge",
+    },
+    "rtorrent": {
+        "url": "http://localhost:8000/RPC2",
+    },
+}
+
+# Test magnet link (Ubuntu ISO - legal, small metadata)
+TEST_MAGNET = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=ubuntu-22.04.3-live-server-amd64.iso"
+
+
+def test_nzbget():
+    """Test NZBGet connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing NZBGet")
+    print("=" * 50)
+
+    url = CONFIG["nzbget"]["url"]
+    username = CONFIG["nzbget"]["username"]
+    password = CONFIG["nzbget"]["password"]
+
+    try:
+        # Test connection via JSON-RPC
+        rpc_url = f"{url}/jsonrpc"
+        response = requests.post(
+            rpc_url,
+            json={"method": "version", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("result", "unknown")
+        print(f"  Connected to NZBGet {version}")
+
+        # Test status
+        response = requests.post(
+            rpc_url,
+            json={"method": "status", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        status = response.json().get("result", {})
+        print(f"  Server state: {'Paused' if status.get('ServerPaused') else 'Running'}")
+        print(f"  Downloads in queue: {status.get('DownloadedSizeMB', 0)} MB downloaded")
+
+        print("  SUCCESS: NZBGet is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to NZBGet")
+        print("  Is the container running? docker ps | grep nzbget")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_sabnzbd():
+    """Test SABnzbd connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing SABnzbd")
+    print("=" * 50)
+
+    url = CONFIG["sabnzbd"]["url"]
+    api_key = CONFIG["sabnzbd"]["api_key"]
+
+    # Try to get API key from config if not set
+    if not api_key:
+        try:
+            import os
+            ini_path = ".local/test-clients/sabnzbd/config/sabnzbd.ini"
+            if os.path.exists(ini_path):
+                with open(ini_path) as f:
+                    for line in f:
+                        if line.startswith("api_key"):
+                            api_key = line.split("=")[1].strip()
+                            print(f"  Found API key in config: {api_key[:8]}...")
+                            break
+        except Exception as e:
+            print(f"  Could not read API key from config: {e}")
+
+    if not api_key:
+        print("  ERROR: No API key configured")
+        print("  Please access http://localhost:8085 and complete initial setup")
+        print("  Then copy the API key from Config > General")
+        return False
+
+    try:
+        # Test connection
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "version", "output": "json"},
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("version", "unknown")
+        print(f"  Connected to SABnzbd {version}")
+
+        # Test queue status
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "queue", "output": "json"},
+            timeout=10,
+        )
+        queue = response.json().get("queue", {})
+        print(f"  Queue status: {queue.get('status', 'unknown')}")
+        print(f"  Items in queue: {len(queue.get('slots', []))}")
+
+        print("  SUCCESS: SABnzbd is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to SABnzbd")
+        print("  Is the container running? docker ps | grep sabnzbd")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_qbittorrent():
+    """Test qBittorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing qBittorrent")
+    print("=" * 50)
+
+    try:
+        import qbittorrentapi
+
+        url = CONFIG["qbittorrent"]["url"]
+        username = CONFIG["qbittorrent"]["username"]
+        password = CONFIG["qbittorrent"]["password"]
+
+        # Parse URL for host/port
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+
+        client = qbittorrentapi.Client(
+            host=parsed.hostname,
+            port=parsed.port or 8080,
+            username=username,
+            password=password,
+        )
+
+        # Test connection
+        client.auth_log_in()
+        version = client.app.version
+        print(f"  Connected to qBittorrent {version}")
+
+        # Get torrent list
+        torrents = client.torrents_info()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        result = client.torrents_add(urls=TEST_MAGNET, is_paused=True)
+        if result == "Ok.":
+            # Wait a moment for it to be added
+            time.sleep(1)
+            torrents = client.torrents_info()
+            if torrents:
+                test_torrent = torrents[-1]  # Most recently added
+                print(f"  Added test torrent: {test_torrent.name[:50]}...")
+                print(f"  Status: {test_torrent.state}")
+
+                # Remove it
+                client.torrents_delete(torrent_hashes=test_torrent.hash, delete_files=True)
+                print("  Removed test torrent")
+        else:
+            print(f"  Add result: {result}")
+
+        print("  SUCCESS: qBittorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: qbittorrent-api not installed")
+        print("  Run: pip install qbittorrent-api")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Forbidden" in str(e) or "401" in str(e):
+            print("\n  Authentication failed. Check password:")
+            print("  1. docker logs test-qbittorrent | grep password")
+            print("  2. Login to http://localhost:8080 and set a known password")
+        return False
+
+
+def test_transmission():
+    """Test Transmission connection."""
+    print("\n" + "=" * 50)
+    print("Testing Transmission")
+    print("=" * 50)
+
+    try:
+        from transmission_rpc import Client
+        from urllib.parse import urlparse
+
+        url = CONFIG["transmission"]["url"]
+        parsed = urlparse(url)
+
+        client = Client(
+            host=parsed.hostname,
+            port=parsed.port or 9091,
+            username=CONFIG["transmission"]["username"],
+            password=CONFIG["transmission"]["password"],
+        )
+
+        # Test connection
+        session = client.get_session()
+        print(f"  Connected to Transmission {session.version}")
+
+        # Get torrent list
+        torrents = client.get_torrents()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent = client.add_torrent(TEST_MAGNET, paused=True)
+        print(f"  Added test torrent: {torrent.name[:50]}...")
+
+        # Get status
+        status = client.get_torrent(torrent.id)
+        print(f"  Status: {status.status} ({status.percent_done * 100:.1f}%)")
+
+        # Remove it
+        client.remove_torrent(torrent.id, delete_data=True)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: Transmission is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: transmission-rpc not installed")
+        print("  Run: pip install transmission-rpc")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_deluge():
+    """Test Deluge Web UI (JSON-RPC) connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing Deluge")
+    print("=" * 50)
+
+    base_url = CONFIG["deluge"]["url"].rstrip("/")
+    password = CONFIG["deluge"]["password"]
+    rpc_url = f"{base_url}/json"
+
+    def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
+        payload = {"id": rpc_id, "method": method, "params": list(params)}
+        resp = session.post(rpc_url, json=payload, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("error"):
+            err = data["error"]
+            if isinstance(err, dict):
+                raise Exception(err.get("message") or str(err))
+            raise Exception(str(err))
+        return data.get("result")
+
+    try:
+        session = requests.Session()
+
+        # Authenticate to Deluge Web
+        if rpc_call(session, 1, "auth.login", password) is not True:
+            raise Exception("Authentication failed (check Deluge Web UI password)")
+
+        # Ensure Deluge Web is connected to a daemon
+        if rpc_call(session, 2, "web.connected") is not True:
+            hosts = rpc_call(session, 3, "web.get_hosts") or []
+            if not hosts:
+                raise Exception(
+                    "Deluge Web UI isn't connected to Deluge core (no hosts configured). "
+                    "Add/connect a daemon in Deluge Web UI → Connection Manager."
+                )
+
+            host_id = hosts[0][0]
+            for entry in hosts:
+                if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
+                    host_id = entry[0]
+                    break
+
+            rpc_call(session, 4, "web.connect", host_id)
+
+            if rpc_call(session, 5, "web.connected") is not True:
+                raise Exception(
+                    "Deluge Web UI couldn't connect to Deluge core. "
+                    "Check Deluge Web UI → Connection Manager."
+                )
+
+        version = rpc_call(session, 6, "daemon.info")
+        print(f"  Connected to Deluge {version}")
+
+        torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
+
+        if torrent_id:
+            torrent_id = str(torrent_id)
+            print(f"  Added test torrent: {torrent_id[:20]}...")
+
+            status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
+            state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
+            progress = status.get("progress", 0) if isinstance(status, dict) else 0
+            print(f"  Status: {state} ({progress:.1f}%)")
+
+            rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
+            print("  Removed test torrent")
+        else:
+            print("  WARNING: Could not add test torrent")
+
+        print("  SUCCESS: Deluge is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to Deluge Web UI")
+        print("  Is the container running? docker ps | grep deluge")
+        return False
+    except requests.exceptions.Timeout:
+        print("  ERROR: Deluge Web UI connection timed out")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "auth" in str(e).lower() or "login" in str(e).lower():
+            print("  Check Deluge Web UI password (default: deluge)")
+        return False
+
+def test_rtorrent():
+    """Test rTorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing rTorrent")
+    print("=" * 50)
+
+    try:
+        import xmlrpc.client
+
+        url = "http://localhost:8000/RPC2"
+        client = xmlrpc.client.ServerProxy(url)
+
+        # Test connection
+        version = client.system.library_version()
+        print(f"  Connected to rTorrent {version}")
+
+        # Get torrent list
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+
+        label = "automated"
+
+        commands = []
+        if label:
+            commands.append(f"d.custom1.set={label}")
+
+        download_dir = "/downloads"
+        if download_dir:
+            commands.append(f"d.directory_base.set={download_dir}")
+
+        # rtorrent is weird in that it doesn't return the torrent ID/hash on add
+        client.load.start("", TEST_MAGNET, ";".join(commands))
+        
+        # but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
+        torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
+        print(f"  Added test torrent: {torrent_id}")
+
+        torrent_list = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal=d.hash=,cat={torrent_id}",
+            "d.hash=",
+            "d.state=",
+            "d.completed_bytes=",
+            "d.size_bytes=",
+            "d.down.rate=",
+            "d.up.rate=",
+            "d.custom1=",
+            "d.complete=",
+        )
+        torrent = torrent_list[0]
+        if not torrent:
+            print("  ERROR: Could not find added torrent in list")
+            return False
+        
+        
+        client.d.erase(torrent_id)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: rTorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: xmlrpc.client not available")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Connection refused" in str(e):
+            print("  Is the container running? docker ps | grep rtorrent")
+        return False
+
+
+def main():
+    print("Download Client Test Suite")
+    print("=" * 50)
+    print("Make sure containers are running:")
+    print("  docker compose -f docker-compose.test-clients.yml up -d")
+
+    results = {}
+
+    # Test usenet clients
+    print("\n" + "=" * 50)
+    print("USENET CLIENTS")
+    print("=" * 50)
+    results["nzbget"] = test_nzbget()
+    results["sabnzbd"] = test_sabnzbd()
+
+    # Test torrent clients
+    print("\n" + "=" * 50)
+    print("TORRENT CLIENTS")
+    print("=" * 50)
+    results["qbittorrent"] = test_qbittorrent()
+    results["transmission"] = test_transmission()
+    results["deluge"] = test_deluge()
+    results["rtorrent"] = test_rtorrent()
+
+    # Summary
+    print("\n" + "=" * 50)
+    print("SUMMARY")
+    print("=" * 50)
+
+    for client, success in results.items():
+        status = "PASS" if success else "FAIL"
+        print(f"  {client}: {status}")
+
+    passed = sum(results.values())
+    total = len(results)
+    print(f"\n  Total: {passed}/{total} passed")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

shelfmark/__init__.py

@@ -0,0 +1 @@
+"""Shelfmark - book search and download service."""

shelfmark/__main__.py

@@ -0,0 +1,8 @@
+"""Package entry point for `python -m shelfmark`."""
+
+from shelfmark.main import app, socketio
+from shelfmark.config.env import FLASK_HOST, FLASK_PORT
+from shelfmark.core.config import config
+
+if __name__ == "__main__":
+    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

shelfmark/api/__init__.py

@@ -0,0 +1 @@
+"""API module - WebSocket handling."""

shelfmark/api/websocket.py

@@ -4,13 +4,14 @@ import logging
 import threading
 from typing import Optional, Dict, Any, Callable, List
 
-from flask_socketio import SocketIO, emit
+from flask_socketio import SocketIO
 
 logger = logging.getLogger(__name__)
 
+
 class WebSocketManager:
     """Manages WebSocket connections and broadcasts."""
-    
+
     def __init__(self):
         self.socketio: Optional[SocketIO] = None
         self._enabled = False
@@ -19,41 +20,29 @@ class WebSocketManager:
         self._on_first_connect_callbacks: List[Callable[[], None]] = []
         self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
         self._needs_rewarm = False  # Flag to trigger warmup callbacks on next connect
-        
+
     def init_app(self, app, socketio: SocketIO):
         """Initialize the WebSocket manager with Flask-SocketIO instance."""
         self.socketio = socketio
         self._enabled = True
         logger.info("WebSocket manager initialized")
-    
+
     def register_on_first_connect(self, callback: Callable[[], None]):
-        """Register a callback to be called when the first client connects.
-        
-        This is useful for warming up resources (like the Cloudflare bypasser)
-        when a user starts using the web UI.
-        """
+        """Register a callback for when the first client connects."""
         self._on_first_connect_callbacks.append(callback)
         logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
-    
-    def register_on_all_disconnect(self, callback: Callable[[], None]):
-        """Register a callback to be called when all clients disconnect.
 
-        This can be used to trigger cleanup or resource release.
-        """
+    def register_on_all_disconnect(self, callback: Callable[[], None]):
+        """Register a callback for when all clients disconnect."""
         self._on_all_disconnect_callbacks.append(callback)
         logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
 
     def request_warmup_on_next_connect(self):
-        """Request that warmup callbacks be triggered on the next client connect.
-
-        This is used when resources (like the Cloudflare bypasser) shut down due to
-        inactivity while clients are still connected. The next connect event should
-        trigger warmup even though it's not technically the "first" connection.
-        """
+        """Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
         with self._connection_lock:
             self._needs_rewarm = True
             logger.debug("Warmup requested for next client connect")
-    
+
     def client_connected(self):
         """Track a new client connection. Call this from the connect event handler."""
         with self._connection_lock:
@@ -79,16 +68,16 @@ class WebSocketManager:
                     thread.start()
                 except Exception as e:
                     logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}")
-    
+
     def client_disconnected(self):
         """Track a client disconnection. Call this from the disconnect event handler."""
         with self._connection_lock:
             self._connection_count = max(0, self._connection_count - 1)
             current_count = self._connection_count
             is_now_zero = current_count == 0
-        
+
         logger.debug(f"Client disconnected. Active connections: {current_count}")
-        
+
         # If all clients have disconnected, trigger cleanup callbacks
         if is_now_zero:
             logger.info("All clients disconnected, triggering disconnect callbacks...")
@@ -97,37 +86,37 @@ class WebSocketManager:
                     callback()
                 except Exception as e:
                     logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}")
-    
+
     def get_connection_count(self) -> int:
         """Get the current number of active WebSocket connections."""
         with self._connection_lock:
             return self._connection_count
-    
+
     def has_active_connections(self) -> bool:
         """Check if there are any active WebSocket connections."""
         return self.get_connection_count() > 0
-        
+
     def is_enabled(self) -> bool:
         """Check if WebSocket is enabled and ready."""
         return self._enabled and self.socketio is not None
-    
+
     def broadcast_status_update(self, status_data: Dict[str, Any]):
         """Broadcast status update to all connected clients."""
         if not self.is_enabled():
             return
-            
+
         try:
             # When calling socketio.emit() outside event handlers, it broadcasts by default
             self.socketio.emit('status_update', status_data)
             logger.debug(f"Broadcasted status update to all clients")
         except Exception as e:
             logger.error(f"Error broadcasting status update: {e}")
-    
+
     def broadcast_download_progress(self, book_id: str, progress: float, status: str):
         """Broadcast download progress update for a specific book."""
         if not self.is_enabled():
             return
-            
+
         try:
             data = {
                 'book_id': book_id,
@@ -139,12 +128,12 @@ class WebSocketManager:
             logger.debug(f"Broadcasted progress for book {book_id}: {progress}%")
         except Exception as e:
             logger.error(f"Error broadcasting download progress: {e}")
-    
+
     def broadcast_notification(self, message: str, notification_type: str = 'info'):
         """Broadcast a notification message to all clients."""
         if not self.is_enabled():
             return
-            
+
         try:
             data = {
                 'message': message,
@@ -156,5 +145,30 @@ class WebSocketManager:
         except Exception as e:
             logger.error(f"Error broadcasting notification: {e}")
 
+    def broadcast_search_status(
+        self,
+        source: str,
+        provider: str,
+        book_id: str,
+        message: str,
+        phase: str = 'searching'
+    ):
+        """Broadcast search status update for a release source search."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'source': source,
+                'provider': provider,
+                'book_id': book_id,
+                'message': message,
+                'phase': phase,
+            }
+            self.socketio.emit('search_status', data)
+        except Exception as e:
+            logger.error(f"Error broadcasting search status: {e}")
+
+
 # Global WebSocket manager instance
 ws_manager = WebSocketManager()

shelfmark/bypass/__init__.py

@@ -0,0 +1,5 @@
+"""Cloudflare bypass utilities."""
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""

shelfmark/bypass/external_bypasser.py

@@ -0,0 +1,128 @@
+"""External Cloudflare bypasser using FlareSolverr."""
+
+import random
+import time
+from threading import Event
+from typing import TYPE_CHECKING, Optional
+
+import requests
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.utils import normalize_http_url
+
+if TYPE_CHECKING:
+    from shelfmark.download import network
+
+logger = setup_logger(__name__)
+
+# Timeout constants (seconds)
+CONNECT_TIMEOUT = 10
+MAX_READ_TIMEOUT = 120
+READ_TIMEOUT_BUFFER = 15
+
+# Retry settings
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
+
+
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service. Returns HTML or None."""
+    raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
+    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
+    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
+
+    bypasser_url = normalize_http_url(raw_bypasser_url)
+    if not bypasser_url or not bypasser_path:
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
+        return None
+
+    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+
+    try:
+        response = requests.post(
+            f"{bypasser_url}{bypasser_path}",
+            headers={"Content-Type": "application/json"},
+            json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
+            timeout=(CONNECT_TIMEOUT, read_timeout)
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+
+        solution = result.get('solution')
+        html = solution.get('response', '') if solution else ''
+
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+
+        return html
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+
+    return None
+
+
+def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
+    """Check if operation was cancelled and raise exception if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(f"External bypasser cancelled {context}")
+        raise BypassCancelledException("Bypass cancelled")
+
+
+def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
+    """Sleep for the specified duration, checking for cancellation each second."""
+    for _ in range(int(seconds)):
+        _check_cancelled(cancel_flag, "during backoff")
+        time.sleep(1)
+    remaining = seconds - int(seconds)
+    if remaining > 0:
+        time.sleep(remaining)
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional["network.AAMirrorSelector"] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML via external bypasser with retries and mirror rotation."""
+    from shelfmark.download import network as network_module
+
+    sel = selector or network_module.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        _check_cancelled(cancel_flag, "by user")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        _sleep_with_cancellation(delay, cancel_flag)
+
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

shelfmark/bypass/fingerprint.py

@@ -0,0 +1,57 @@
+"""Browser fingerprint profile management for bypass stealth."""
+
+import random
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+COMMON_RESOLUTIONS = [
+    (1920, 1080, 0.35),  
+    (1366, 768, 0.18),   
+    (1536, 864, 0.10),   
+    (1440, 900, 0.08),   
+    (1280, 720, 0.07),   
+    (1600, 900, 0.06),   
+    (1280, 800, 0.05),   
+    (2560, 1440, 0.04),  
+    (1680, 1050, 0.04),  
+    (1920, 1200, 0.03),  
+]
+
+# Current screen size (module-level singleton)
+_current_screen_size: Optional[tuple[int, int]] = None
+
+
+def get_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    if _current_screen_size is None:
+        _current_screen_size = _generate_screen_size()
+        logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
+    return _current_screen_size
+
+
+def rotate_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    old_size = _current_screen_size
+    _current_screen_size = _generate_screen_size()
+    width, height = _current_screen_size
+
+    if old_size:
+        logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
+    else:
+        logger.info(f"Generated screen size: {width}x{height}")
+
+    return _current_screen_size
+
+
+def clear_screen_size() -> None:
+    global _current_screen_size
+    _current_screen_size = None
+
+
+def _generate_screen_size() -> tuple[int, int]:
+    resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
+    weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
+    return random.choices(resolutions, weights=weights)[0]

shelfmark/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration module - environment variables and settings."""

shelfmark/config/booklore_settings.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.outputs.booklore import (
+    BookloreConfig,
+    BookloreError,
+    booklore_list_libraries,
+    booklore_login,
+)
+
+logger = setup_logger(__name__)
+
+_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
+    "key": None,
+    "library_options": [],
+    "path_options": [],
+}
+
+
+def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
+    return f"{base_url}|{username}|{hash(password)}"
+
+
+def _get_booklore_select_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    # library_id/path_id are not used for login/library listing
+    booklore_config = BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=1,
+        path_id=1,
+        verify_tls=True,
+        refresh_after_upload=True,
+    )
+
+    token = booklore_login(booklore_config)
+    libraries = booklore_list_libraries(booklore_config, token) or []
+    logger.debug("Booklore libraries response: %s", libraries)
+
+    library_options: list[dict[str, Any]] = []
+    path_options: list[dict[str, Any]] = []
+
+    for library in libraries:
+        if not isinstance(library, dict):
+            continue
+
+        library_id = library.get("id")
+        if library_id is None:
+            continue
+
+        library_name = str(library.get("name") or f"Library {library_id}")
+        library_id_str = str(library_id)
+
+        library_options.append({"value": library_id_str, "label": library_name})
+
+        paths = library.get("paths") or []
+        if not isinstance(paths, list):
+            continue
+
+        for path in paths:
+            if not isinstance(path, dict):
+                continue
+
+            path_id = path.get("id")
+            if path_id is None:
+                continue
+
+            path_label = str(path.get("path") or f"Path {path_id}")
+            path_options.append(
+                {
+                    "value": str(path_id),
+                    "label": f"{library_name}: {path_label}",
+                    "childOf": library_id_str,
+                }
+            )
+
+    logger.debug(
+        "Booklore options built: libraries=%d paths=%d",
+        len(library_options),
+        len(path_options),
+    )
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    _BOOKLORE_OPTIONS_CACHE.update(
+        {
+            "key": cache_key,
+            "library_options": library_options,
+            "path_options": path_options,
+        }
+    )
+
+    return library_options, path_options
+
+
+def _get_booklore_cached_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+        return (
+            _BOOKLORE_OPTIONS_CACHE.get("library_options", []),
+            _BOOKLORE_OPTIONS_CACHE.get("path_options", []),
+        )
+
+    return _get_booklore_select_options(base_url, username, password)
+
+
+def get_booklore_library_options() -> list[dict[str, Any]]:
+    """Build Booklore library options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        library_options, _ = _get_booklore_cached_options(base_url, username, password)
+        return library_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore libraries: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
+        return []
+
+
+def get_booklore_path_options() -> list[dict[str, Any]]:
+    """Build Booklore path options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        _, path_options = _get_booklore_cached_options(base_url, username, password)
+        return path_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore paths: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
+        return []
+
+
+def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test the Booklore connection using current form values."""
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
+    password = _get_value("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        return {"success": False, "message": "Booklore URL is required"}
+    if not username:
+        return {"success": False, "message": "Booklore username is required"}
+    if not password:
+        return {"success": False, "message": "Booklore password is required"}
+
+    try:
+        library_options, _ = _get_booklore_select_options(base_url, username, password)
+
+        message = "Connected to Booklore"
+        if library_options:
+            message = f"Connected to Booklore ({len(library_options)} libraries)"
+
+        return {"success": True, "message": message}
+    except BookloreError as exc:
+        return {"success": False, "message": str(exc)}

shelfmark/config/env.py

@@ -0,0 +1,153 @@
+"""Bootstrap environment variables. No local dependencies - import first."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+
+def string_to_bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ["true", "yes", "1", "y"]
+
+
+def _read_debug_from_config() -> bool:
+    """Read DEBUG from env var or config file (import-time safe)."""
+    env_debug = os.environ.get("DEBUG")
+    if env_debug is not None:
+        return string_to_bool(env_debug)
+
+    # Try to read from config file
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    config_file = config_dir / "plugins" / "advanced.json"
+
+    if config_file.exists():
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+                if "DEBUG" in config:
+                    return bool(config["DEBUG"])
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return False
+
+
+def _is_sqlite_file(path: Path) -> bool:
+    """Check if a file is a valid SQLite database by reading magic bytes."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(16)
+            return header[:16] == b"SQLite format 3\x00"
+    except (OSError, PermissionError):
+        return False
+
+
+def _resolve_cwa_db_path() -> Path | None:
+    """Resolve CWA database path from env var or default location."""
+    env_path = os.getenv("CWA_DB_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file() and _is_sqlite_file(path):
+            return path
+
+    # Check default mount path
+    default_path = Path("/auth/app.db")
+    if default_path.exists() and default_path.is_file() and _is_sqlite_file(default_path):
+        return default_path
+
+    return None
+
+
+def _is_config_dir_writable() -> bool:
+    """Check if the config directory exists and is writable."""
+    try:
+        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
+            return False
+        test_file = CONFIG_DIR / ".write_test"
+        test_file.touch()
+        test_file.unlink()
+        return True
+    except (OSError, PermissionError):
+        return False
+
+
+def is_covers_cache_enabled() -> bool:
+    """Check if cover caching is enabled (requires setting + writable config dir)."""
+    from shelfmark.core.config import config
+    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
+    return setting_enabled and _is_config_dir_writable()
+
+
+# =============================================================================
+# Bootstrap paths - needed before settings registry is available
+# =============================================================================
+
+CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
+LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
+LOG_DIR = LOG_ROOT / "shelfmark"
+LOG_FILE = LOG_DIR / "shelfmark.log"
+TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
+INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
+
+
+# =============================================================================
+# Logger configuration - needed before settings registry is available
+# =============================================================================
+
+DEBUG = _read_debug_from_config()
+LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
+ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+
+
+# =============================================================================
+# Flask configuration - needed before app starts
+# =============================================================================
+
+FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
+
+
+# =============================================================================
+# Authentication
+# =============================================================================
+
+SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
+CWA_DB_PATH = _resolve_cwa_db_path()
+
+
+# =============================================================================
+# Version information from Docker build
+# =============================================================================
+
+BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
+RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
+
+
+# =============================================================================
+# Capability detection - runtime checks, not user-configurable
+# =============================================================================
+
+DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
+TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
+USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
+
+
+# =============================================================================
+# Debug/development settings
+# =============================================================================
+
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
+
+
+# =============================================================================
+# Legacy migration support - will be removed in future version
+# =============================================================================
+
+# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
+# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
+_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))

shelfmark/config/security.py

@@ -0,0 +1,290 @@
+"""Authentication settings registration."""
+
+from typing import Any, Dict
+
+from werkzeug.security import generate_password_hash
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    register_settings,
+    register_on_save,
+    load_config_file,
+    TextField,
+    SelectField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+
+logger = setup_logger(__name__)
+
+
+def _migrate_security_settings() -> None:
+    import json
+    from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
+
+    try:
+        config = load_config_file("security")
+        migrated = False
+        
+        # Migrate USE_CWA_AUTH to AUTH_METHOD
+        if "USE_CWA_AUTH" in config:
+            old_value = config.pop("USE_CWA_AUTH")
+            
+            # Only set AUTH_METHOD if it doesn't already exist
+            if "AUTH_METHOD" not in config:
+                if old_value:
+                    config["AUTH_METHOD"] = "cwa"
+                    logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
+                else:
+                    # If USE_CWA_AUTH was False, determine auth method from credentials
+                    if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                        config["AUTH_METHOD"] = "builtin"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
+                    else:
+                        config["AUTH_METHOD"] = "none"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
+                migrated = True
+            else:
+                logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
+                migrated = True
+        
+        # Migrate RESTRICT_SETTINGS_TO_ADMIN to CWA_RESTRICT_SETTINGS_TO_ADMIN
+        if "RESTRICT_SETTINGS_TO_ADMIN" in config:
+            old_value = config.pop("RESTRICT_SETTINGS_TO_ADMIN")
+            
+            # Only migrate if new key doesn't exist
+            if "CWA_RESTRICT_SETTINGS_TO_ADMIN" not in config:
+                config["CWA_RESTRICT_SETTINGS_TO_ADMIN"] = old_value
+                logger.info(f"Migrated RESTRICT_SETTINGS_TO_ADMIN={old_value} to CWA_RESTRICT_SETTINGS_TO_ADMIN={old_value}")
+                migrated = True
+            else:
+                logger.info("Removed deprecated RESTRICT_SETTINGS_TO_ADMIN setting (CWA_RESTRICT_SETTINGS_TO_ADMIN already exists)")
+                migrated = True
+        
+        # Save config if any migrations occurred
+        if migrated:
+            _ensure_config_dir("security")
+            config_path = _get_config_file_path("security")
+            with open(config_path, 'w') as f:
+                json.dump(config, f, indent=2)
+            logger.info("Security settings migration completed successfully")
+        else:
+            logger.debug("No security settings migration needed")
+    
+    except FileNotFoundError:
+        logger.debug("No existing security config file found - nothing to migrate")
+    except Exception as e:
+        logger.error(f"Failed to migrate security settings: {e}")
+
+
+def _clear_builtin_credentials() -> Dict[str, Any]:
+    """Clear built-in credentials to allow public access."""
+    import json
+    from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
+
+    try:
+        config = load_config_file("security")
+        config.pop("BUILTIN_USERNAME", None)
+        config.pop("BUILTIN_PASSWORD_HASH", None)
+
+        _ensure_config_dir("security")
+        config_path = _get_config_file_path("security")
+        with open(config_path, 'w') as f:
+            json.dump(config, f, indent=2)
+
+        logger.info("Cleared credentials")
+        return {"success": True, "message": "Credentials cleared. The app is now publicly accessible."}
+
+    except Exception as e:
+        logger.error(f"Failed to clear credentials: {e}")
+        return {"success": False, "message": f"Failed to clear credentials: {str(e)}"}
+
+
+def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Custom save handler for security settings.
+
+    Handles password validation and hashing:
+    - If new password is provided, validate confirmation and hash it
+    - If password fields are empty, preserve existing hash
+    - Never store raw passwords
+    - Ensure username is present if password is set
+
+    Returns:
+        Dict with processed values to save and any validation errors.
+    """
+    password = values.get("BUILTIN_PASSWORD", "")
+    password_confirm = values.get("BUILTIN_PASSWORD_CONFIRM", "")
+
+    # Remove raw password fields - they should never be persisted
+    values.pop("BUILTIN_PASSWORD", None)
+    values.pop("BUILTIN_PASSWORD_CONFIRM", None)
+
+    # If password is provided, validate and hash it
+    if password:
+        if not values.get("BUILTIN_USERNAME"):
+            return {
+                "error": True,
+                "message": "Username cannot be empty",
+                "values": values
+            }
+
+        if password != password_confirm:
+            return {
+                "error": True,
+                "message": "Passwords do not match",
+                "values": values
+            }
+
+        if len(password) < 4:
+            return {
+                "error": True,
+                "message": "Password must be at least 4 characters",
+                "values": values
+            }
+
+        # Hash the password
+        values["BUILTIN_PASSWORD_HASH"] = generate_password_hash(password)
+        logger.info("Password hash updated")
+
+    # If no password provided but username is being set, preserve existing hash
+    elif "BUILTIN_USERNAME" in values:
+        existing = load_config_file("security")
+        if "BUILTIN_PASSWORD_HASH" in existing:
+            values["BUILTIN_PASSWORD_HASH"] = existing["BUILTIN_PASSWORD_HASH"]
+
+    return {"error": False, "values": values}
+
+
+@register_settings("security", "Security", icon="shield", order=5)
+def security_settings():  
+    """Security and authentication settings."""
+    from shelfmark.config.env import CWA_DB_PATH
+
+    cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
+
+    auth_method_options = [
+        {"label": "No Authentication", "value": "none"},
+        {"label": "Username/Password", "value": "builtin"},
+        {"label": "Proxy Authentication", "value": "proxy"},
+    ]
+    if cwa_db_available:
+        auth_method_options.append({"label": "Calibre-Web Database", "value": "cwa"})
+
+    auth_method_description = "Select the authentication method for accessing Shelfmark."
+    if not cwa_db_available:
+        auth_method_description += " Calibre-Web database option requires mounting your Calibre-Web app.db to /auth/app.db."
+
+    fields = [
+        SelectField(
+            key="AUTH_METHOD",
+            label="Authentication Method",
+            description=auth_method_description,
+            options=auth_method_options,
+            default="none",
+            env_supported=False,
+        ),
+        TextField(
+            key="BUILTIN_USERNAME",
+            label="Username",
+            description="Set a username and password to require login. Leave both empty for public access.",
+            placeholder="Enter username",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        PasswordField(
+            key="BUILTIN_PASSWORD",
+            label="Set Password",
+            description="Fill in to set or change the password.",
+            placeholder="Enter new password",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        PasswordField(
+            key="BUILTIN_PASSWORD_CONFIRM",
+            label="Confirm Password",
+            placeholder="Confirm new password",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        ActionButton(
+            key="clear_credentials",
+            label="Clear Credentials",
+            description="Remove login requirement and make the app publicly accessible.",
+            style="danger",
+            callback=_clear_builtin_credentials,
+            show_when={"field": "AUTH_METHOD", "value": "builtin"},
+        ),
+        TextField(
+            key="PROXY_AUTH_USER_HEADER",
+            label="Proxy Auth User Header",
+            description=(
+                "The HTTP header your proxy uses to pass the authenticated username."
+            ),
+            placeholder="e.g. X-Auth-User",
+            default="X-Auth-User",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        TextField(
+            key="PROXY_AUTH_LOGOUT_URL",
+            label="Proxy Auth Logout URL",
+            description=(
+                "The URL to redirect users to for logging out."
+                " Leave empty to disable logout functionality."
+            ),
+            placeholder="https://myauth.example.com/logout",
+            default="",
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        CheckboxField(
+            key="PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
+            label="Restrict Settings to Admins authenticated via Proxy",
+            description=(
+                "Only users in the admin group can access settings."
+            ),
+            default=False,
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "proxy"},
+        ),
+        TextField(
+            key="PROXY_AUTH_ADMIN_GROUP_HEADER",
+            label="Proxy Auth Admin Group Header",
+            description=(
+                "The HTTP header your proxy uses to pass the user's groups/roles."
+            ),
+            placeholder="e.g. X-Auth-Groups",
+            default="X-Auth-Groups",
+            env_supported=False,
+            show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
+        ),
+        TextField(
+            key="PROXY_AUTH_ADMIN_GROUP_NAME",
+            label="Proxy Auth Admin Group Name",
+            description=(
+                "The name of the group/role that should have admin access."
+            ),
+            placeholder="e.g. admins",
+            default="admins",
+            env_supported=False,
+            show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
+        ),
+        CheckboxField(
+            key="CWA_RESTRICT_SETTINGS_TO_ADMIN",
+            label="Restrict Settings to Admins authenticated via Calibre-Web",
+            description=(
+                "Only users with admin role in Calibre-Web can access settings."
+            ),
+            default=False,
+            env_supported=False,
+            show_when={"field": "AUTH_METHOD", "value": "cwa"},
+        ),
+    ]
+
+    return fields
+
+
+# Register the on_save handler for this tab
+register_on_save("security", _on_save_security)

shelfmark/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core module - shared models, queue, and utilities."""
+
+from shelfmark.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
+from shelfmark.core.queue import BookQueue, book_queue
+from shelfmark.core.logger import setup_logger

shelfmark/core/cache.py

@@ -0,0 +1,172 @@
+"""Thread-safe in-memory cache with TTL support."""
+
+import threading
+import time
+from dataclasses import dataclass
+from functools import wraps
+from typing import Any, Callable, Dict, Optional, TypeVar
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+T = TypeVar("T")
+
+
+@dataclass
+class CacheEntry:
+    """A cached value with expiration time."""
+    value: Any
+    expires_at: float
+
+
+class CacheService:
+    """Thread-safe in-memory cache with TTL support."""
+
+    def __init__(self, max_size: int = 1000):
+        """Initialize cache with max_size entries before eviction."""
+        self._cache: Dict[str, CacheEntry] = {}
+        self._lock = threading.Lock()
+        self._max_size = max_size
+
+    def get(self, key: str) -> Optional[Any]:
+        """Get cached value if not expired."""
+        with self._lock:
+            entry = self._cache.get(key)
+            if entry is None:
+                return None
+
+            if time.time() > entry.expires_at:
+                del self._cache[key]
+                return None
+
+            return entry.value
+
+    def set(self, key: str, value: Any, ttl: int) -> None:
+        """Cache value with TTL in seconds."""
+        with self._lock:
+            # Evict oldest entries if at capacity
+            if len(self._cache) >= self._max_size:
+                self._evict_oldest()
+
+            self._cache[key] = CacheEntry(
+                value=value,
+                expires_at=time.time() + ttl
+            )
+
+    def invalidate(self, key: str) -> bool:
+        """Remove specific cache entry. Returns True if found."""
+        with self._lock:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        with self._lock:
+            self._cache.clear()
+
+    def cleanup_expired(self) -> int:
+        """Remove all expired entries. Returns count removed."""
+        with self._lock:
+            now = time.time()
+            expired_keys = [
+                key for key, entry in self._cache.items()
+                if entry.expires_at < now
+            ]
+            for key in expired_keys:
+                del self._cache[key]
+            return len(expired_keys)
+
+    def _evict_oldest(self) -> None:
+        """Evict ~10% of oldest entries. Called with lock held."""
+        if not self._cache:
+            return
+
+        # Remove ~10% of entries, oldest first
+        entries_to_remove = max(1, len(self._cache) // 10)
+        sorted_entries = sorted(
+            self._cache.items(),
+            key=lambda x: x[1].expires_at
+        )
+
+        for key, _ in sorted_entries[:entries_to_remove]:
+            del self._cache[key]
+
+    def stats(self) -> Dict[str, int]:
+        """Get cache statistics (size, max_size)."""
+        with self._lock:
+            return {
+                "size": len(self._cache),
+                "max_size": self._max_size
+            }
+
+
+# Global cache instance for metadata providers
+_metadata_cache = CacheService(max_size=1000)
+
+
+def get_metadata_cache() -> CacheService:
+    """Get the global metadata cache instance."""
+    return _metadata_cache
+
+
+def cache_key(*args, **kwargs) -> str:
+    """Generate cache key from arguments."""
+    parts = [str(arg) for arg in args]
+    parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
+    return ":".join(parts)
+
+
+def cacheable(
+    ttl: Optional[int] = None,
+    ttl_key: Optional[str] = None,
+    ttl_default: int = 300,
+    key_prefix: str = ""
+):
+    """Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            # Check if metadata caching is enabled
+            from shelfmark.core.config import config
+
+            if not config.get("METADATA_CACHE_ENABLED", True):
+                # Caching disabled, execute function directly
+                return func(*args, **kwargs)
+
+            # Determine TTL: static or from config
+            if ttl is not None:
+                effective_ttl = ttl
+            elif ttl_key:
+                effective_ttl = config.get(ttl_key, ttl_default)
+            else:
+                effective_ttl = ttl_default
+
+            # Generate cache key from function name and arguments
+            # Skip 'self' argument if present (first arg of method)
+            cache_args = args[1:] if args and hasattr(args[0], func.__name__) else args
+
+            key = cache_key(
+                key_prefix or func.__name__,
+                *cache_args,
+                **kwargs
+            )
+
+            # Check cache
+            cached = _metadata_cache.get(key)
+            if cached is not None:
+                return cached
+
+            # Execute function and cache result
+            result = func(*args, **kwargs)
+
+            # Only cache non-None results
+            if result is not None:
+                _metadata_cache.set(key, result, effective_ttl)
+
+            return result
+
+        return wrapper
+    return decorator

shelfmark/core/config.py

@@ -0,0 +1,183 @@
+"""Configuration singleton with ENV > config file > default resolution."""
+
+from threading import Lock
+from typing import Any, Dict, Optional
+
+# Import lazily to avoid circular imports
+_registry_module = None
+_env_module = None
+
+
+def _get_registry():
+    """Lazy import of settings registry to avoid circular imports."""
+    global _registry_module
+    if _registry_module is None:
+        from shelfmark.core import settings_registry
+        _registry_module = settings_registry
+    return _registry_module
+
+
+def _get_env():
+    """Lazy import of env module for fallback values."""
+    global _env_module
+    if _env_module is None:
+        from shelfmark.config import env
+        _env_module = env
+    return _env_module
+
+
+class Config:
+    """
+    Dynamic configuration singleton that provides live settings access.
+
+    Settings are resolved with priority: ENV var > config file > default.
+    Values are cached for performance and can be refreshed when settings change.
+    """
+
+    _instance: Optional['Config'] = None
+    _lock = Lock()
+
+    def __new__(cls) -> 'Config':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+        self._cache: Dict[str, Any] = {}
+        self._field_map: Dict[str, tuple] = {}  # key -> (field, tab_name)
+        self._cache_lock = Lock()
+        self._initialized = True
+        self._loaded = False
+
+    def _ensure_loaded(self) -> None:
+        """Ensure settings are loaded from the registry."""
+        if self._loaded:
+            return
+        with self._cache_lock:
+            if self._loaded:
+                return
+            self._load_settings()
+
+    def _load_settings(self) -> None:
+        """Load all settings from the registry."""
+        # Ensure all settings modules are imported before loading
+        # This handles cases where config is accessed before settings are registered
+        try:
+            import shelfmark.config.settings  # noqa: F401 - main app settings
+            import shelfmark.release_sources  # noqa: F401 - plugin settings
+            import shelfmark.metadata_providers  # noqa: F401 - plugin settings
+        except ImportError:
+            pass
+
+        registry = _get_registry()
+
+        # On first load, sync ENV values to config files
+        # This ensures ENV values persist even if ENV vars are later removed
+        if not hasattr(self, '_env_synced'):
+            registry.sync_env_to_config()
+            self._env_synced = True
+
+        # Build field map from all registered tabs
+        self._field_map.clear()
+        self._cache.clear()
+
+        for tab in registry.get_all_settings_tabs():
+            for field in tab.fields:
+                # Skip action buttons and headings - they don't have values
+                if isinstance(field, (registry.ActionButton, registry.HeadingField)):
+                    continue
+
+                key = field.key
+                self._field_map[key] = (field, tab.name)
+
+                # Load current value
+                value = registry.get_setting_value(field, tab.name)
+                self._cache[key] = value
+
+        self._loaded = True
+
+    def refresh(self) -> None:
+        """
+        Refresh all cached settings from config files.
+
+        Call this after settings are updated via the UI to ensure
+        the config singleton reflects the new values.
+        """
+        with self._cache_lock:
+            self._loaded = False
+            self._load_settings()
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """
+        Get a setting value by key.
+
+        Args:
+            key: The setting key (e.g., 'MAX_RETRY')
+            default: Default value if setting not found
+
+        Returns:
+            The setting value, or default if not found
+        """
+        self._ensure_loaded()
+        return self._cache.get(key, default)
+
+    def __getattr__(self, name: str) -> Any:
+        """
+        Allow attribute-style access to settings.
+
+        Example: config.MAX_RETRY instead of config.get('MAX_RETRY')
+        """
+        # Avoid recursion for internal attributes
+        if name.startswith('_'):
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
+
+        self._ensure_loaded()
+
+        if name in self._cache:
+            return self._cache[name]
+
+        # Fallback to env module for settings not in registry
+        # This ensures backward compatibility during migration
+        env = _get_env()
+        if hasattr(env, name):
+            return getattr(env, name)
+
+        raise AttributeError(f"Setting '{name}' not found in config or env")
+
+    def is_from_env(self, key: str) -> bool:
+        """
+        Check if a setting's value comes from an environment variable.
+
+        Args:
+            key: The setting key
+
+        Returns:
+            True if the value is set via ENV var, False otherwise
+        """
+        self._ensure_loaded()
+
+        if key not in self._field_map:
+            return False
+
+        field, _ = self._field_map[key]
+        registry = _get_registry()
+        return registry.is_value_from_env(field)
+
+    def get_all(self) -> Dict[str, Any]:
+        """
+        Get all cached settings as a dictionary.
+
+        Returns:
+            Dict of all setting keys to their current values
+        """
+        self._ensure_loaded()
+        return dict(self._cache)
+
+
+# Global singleton instance
+config = Config()

shelfmark/core/image_cache.py

@@ -0,0 +1,569 @@
+"""Disk-based image cache with LRU eviction."""
+
+import json
+import os
+import threading
+import time
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Image type detection via magic bytes
+IMAGE_SIGNATURES = {
+    b'\xff\xd8\xff': ('image/jpeg', 'jpg'),
+    b'\x89PNG\r\n\x1a\n': ('image/png', 'png'),
+    b'GIF87a': ('image/gif', 'gif'),
+    b'GIF89a': ('image/gif', 'gif'),
+    b'RIFF': ('image/webp', 'webp'),  # WebP starts with RIFF
+}
+
+# HTTP headers for image fetching
+FETCH_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36',
+    'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+}
+
+# Maximum image size to fetch (5 MB)
+MAX_IMAGE_SIZE = 5 * 1024 * 1024
+
+# Negative cache TTL (for failed fetches) - 1 hour
+NEGATIVE_CACHE_TTL = 3600
+
+# Transient failure cache TTL (for timeouts/connection errors) - 60 seconds
+# Short enough to retry soon, long enough to prevent spam during one page view
+TRANSIENT_CACHE_TTL = 60
+
+
+def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
+    """Detect image type from magic bytes.
+
+    Args:
+        data: Image data bytes
+
+    Returns:
+        Tuple of (content_type, extension) or None if not recognized
+    """
+    for signature, (content_type, ext) in IMAGE_SIGNATURES.items():
+        if data.startswith(signature):
+            return content_type, ext
+
+    # Special case for WebP - check for WEBP after RIFF
+    if data.startswith(b'RIFF') and len(data) > 12 and data[8:12] == b'WEBP':
+        return 'image/webp', 'webp'
+
+    return None
+
+
+class ImageCacheService:
+    """Persistent image cache with LRU eviction and TTL support."""
+
+    def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0):
+        """Initialize the image cache.
+
+        Args:
+            cache_dir: Directory to store cached images
+            max_size_mb: Maximum cache size in megabytes
+            ttl_seconds: Time-to-live in seconds (0 = forever)
+        """
+        self.cache_dir = cache_dir
+        self.max_size_bytes = max_size_mb * 1024 * 1024
+        self.ttl_seconds = ttl_seconds
+        self.index_path = cache_dir / "cache_index.json"
+        self._lock = threading.RLock()
+        self._index: Dict[str, Dict[str, Any]] = {}
+
+        # Stats tracking
+        self._hits = 0
+        self._misses = 0
+
+        # Ensure cache directory exists
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load existing index and sync with files on disk (once at startup)
+        self._load_index()
+        self._sync_index_with_files()
+
+    def _load_index(self) -> None:
+        """Load cache index from disk."""
+        if not self.index_path.exists():
+            self._index = {}
+            return
+
+        try:
+            with open(self.index_path, 'r') as f:
+                self._index = json.load(f)
+        except (json.JSONDecodeError, IOError):
+            self._index = {}
+
+    def _sync_index_with_files(self) -> None:
+        """Sync cache index with actual files on disk.
+
+        - Adds entries for files that exist but aren't in index
+        - Removes entries for files that no longer exist (non-negative only)
+        - Preserves negative cache entries (they have no files)
+        """
+        image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
+        added_count = 0
+        removed_count = 0
+
+        # Build set of files that exist on disk
+        existing_files: Dict[str, Path] = {}
+        for file_path in self.cache_dir.iterdir():
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() not in image_extensions:
+                continue
+            existing_files[file_path.stem] = file_path
+
+        # Add files that aren't in the index
+        for cache_id, file_path in existing_files.items():
+            if cache_id in self._index:
+                continue
+
+            ext = file_path.suffix.lstrip('.')
+            stat = file_path.stat()
+
+            # Detect content type
+            try:
+                with open(file_path, 'rb') as f:
+                    header = f.read(16)
+                detected = _detect_image_type(header)
+                content_type = detected[0] if detected else f'image/{ext}'
+            except IOError:
+                content_type = f'image/{ext}'
+
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': stat.st_size,
+                'cached_at': stat.st_mtime,
+                'accessed_at': stat.st_mtime,
+            }
+            added_count += 1
+
+        # Remove index entries for missing files (skip negative cache entries)
+        stale_entries = []
+        for cache_id, entry in self._index.items():
+            if entry.get('negative', False):
+                continue  # Negative entries don't have files
+            if cache_id not in existing_files:
+                stale_entries.append(cache_id)
+
+        for cache_id in stale_entries:
+            del self._index[cache_id]
+            removed_count += 1
+
+        if added_count > 0 or removed_count > 0:
+            self._save_index()
+
+    def _save_index(self) -> None:
+        """Save cache index to disk."""
+        try:
+            # Write to temp file first, then rename for atomicity
+            temp_path = self.index_path.with_suffix('.tmp')
+            with open(temp_path, 'w') as f:
+                json.dump(self._index, f)
+            temp_path.rename(self.index_path)
+        except IOError:
+            pass
+
+    def _get_image_path(self, cache_id: str, ext: str) -> Path:
+        """Get the file path for a cached image."""
+        return self.cache_dir / f"{cache_id}.{ext}"
+
+    def _is_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a cache entry is expired."""
+        if self.ttl_seconds == 0:
+            return False
+        return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
+
+    def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a negative cache entry is expired.
+
+        Transient failures (timeouts) expire after TRANSIENT_CACHE_TTL (60s).
+        Permanent failures (404s) expire after NEGATIVE_CACHE_TTL (1 hour).
+        """
+        if not entry.get('negative', False):
+            return False
+
+        cached_at = entry.get('cached_at', 0)
+        ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
+        return (time.time() - cached_at) > ttl
+
+    def _calculate_total_size(self) -> int:
+        """Calculate total size of cached images."""
+        return sum(entry.get('size', 0) for entry in self._index.values())
+
+    def _evict_if_needed(self, required_space: int = 0) -> None:
+        """Evict old entries if cache is over size limit.
+
+        Uses LRU eviction based on accessed_at timestamp.
+        """
+        current_size = self._calculate_total_size()
+        target_size = self.max_size_bytes - required_space
+
+        if current_size <= target_size:
+            return
+
+        # Sort entries by accessed_at (oldest first)
+        sorted_entries = sorted(
+            self._index.items(),
+            key=lambda x: x[1].get('accessed_at', 0)
+        )
+
+        evicted_count = 0
+        for cache_id, entry in sorted_entries:
+            if current_size <= target_size:
+                break
+
+            # Delete the image file
+            ext = entry.get('ext', 'jpg')
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                if image_path.exists():
+                    image_path.unlink()
+            except IOError:
+                pass
+
+            # Update tracking
+            current_size -= entry.get('size', 0)
+            del self._index[cache_id]
+            evicted_count += 1
+
+        if evicted_count > 0:
+            self._save_index()
+
+    def get(self, cache_id: str) -> Optional[Tuple[bytes, str]]:
+        """Get a cached image.
+
+        Args:
+            cache_id: Cache key (book ID or composite key)
+
+        Returns:
+            Tuple of (image_data, content_type) or None if not cached/expired
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+
+            # Try reloading from disk if not found (handles multiprocess case)
+            if not entry:
+                self._load_index()
+                entry = self._index.get(cache_id)
+                if not entry:
+                    self._misses += 1
+                    return None
+
+            # Check for negative cache (failed fetch)
+            if entry.get('negative', False):
+                if self._is_negative_expired(entry):
+                    # Negative cache expired, allow retry
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+                # Still in negative cache, return None (don't retry)
+                return None
+
+            # Check for expired entry
+            if self._is_expired(entry):
+                # Remove expired entry
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+                del self._index[cache_id]
+                self._save_index()
+                self._misses += 1
+                return None
+
+            # Try to read the cached image
+            ext = entry.get('ext', 'jpg')
+            content_type = entry.get('content_type', 'image/jpeg')
+            image_path = self._get_image_path(cache_id, ext)
+
+            try:
+                if not image_path.exists():
+                    # File missing, remove from index
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+
+                with open(image_path, 'rb') as f:
+                    data = f.read()
+
+                # Update accessed time
+                entry['accessed_at'] = time.time()
+                self._save_index()
+
+                self._hits += 1
+                return data, content_type
+
+            except IOError:
+                self._misses += 1
+                return None
+
+    def put(self, cache_id: str, data: bytes, content_type: str) -> bool:
+        """Store an image in the cache.
+
+        Args:
+            cache_id: Cache key
+            data: Image data bytes
+            content_type: MIME type of the image
+
+        Returns:
+            True if stored successfully
+        """
+        with self._lock:
+            # Detect image type for extension
+            detected = _detect_image_type(data)
+            if detected:
+                content_type, ext = detected
+            else:
+                # Fall back to content-type header
+                if 'jpeg' in content_type or 'jpg' in content_type:
+                    ext = 'jpg'
+                elif 'png' in content_type:
+                    ext = 'png'
+                elif 'gif' in content_type:
+                    ext = 'gif'
+                elif 'webp' in content_type:
+                    ext = 'webp'
+                else:
+                    ext = 'jpg'  # Default
+
+            image_size = len(data)
+
+            # Evict if needed to make room
+            self._evict_if_needed(image_size)
+
+            # Write image to disk
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                with open(image_path, 'wb') as f:
+                    f.write(data)
+            except IOError:
+                return False
+
+            # Update index
+            now = time.time()
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': image_size,
+                'cached_at': now,
+                'accessed_at': now,
+                'negative': False,
+            }
+            self._save_index()
+            return True
+
+    def put_negative(self, cache_id: str, transient: bool = False) -> None:
+        """Store a negative cache entry (failed fetch).
+
+        Args:
+            cache_id: Cache key
+            transient: If True, uses shorter TTL (for timeouts/connection errors)
+        """
+        with self._lock:
+            self._index[cache_id] = {
+                'negative': True,
+                'transient': transient,
+                'cached_at': time.time(),
+            }
+            self._save_index()
+
+    def delete(self, cache_id: str) -> bool:
+        """Delete a single cache entry.
+
+        Args:
+            cache_id: Cache key
+
+        Returns:
+            True if entry existed and was deleted
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+            if not entry:
+                return False
+
+            # Delete file if it exists
+            if not entry.get('negative', False):
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+
+            del self._index[cache_id]
+            self._save_index()
+            return True
+
+    def clear(self) -> int:
+        """Clear all cached images.
+
+        Returns:
+            Number of entries cleared
+        """
+        with self._lock:
+            count = len(self._index)
+
+            # Delete all image files
+            for cache_id, entry in self._index.items():
+                if not entry.get('negative', False):
+                    ext = entry.get('ext', 'jpg')
+                    image_path = self._get_image_path(cache_id, ext)
+                    try:
+                        if image_path.exists():
+                            image_path.unlink()
+                    except IOError:
+                        pass
+
+            # Clear index
+            self._index = {}
+            self._save_index()
+
+            # Reset stats
+            self._hits = 0
+            self._misses = 0
+
+            return count
+
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dict with size, count, hit rate, etc.
+        """
+        with self._lock:
+            total_size = self._calculate_total_size()
+            entry_count = len(self._index)
+            negative_count = sum(1 for e in self._index.values() if e.get('negative', False))
+            total_requests = self._hits + self._misses
+            hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0
+
+            return {
+                'entry_count': entry_count,
+                'negative_count': negative_count,
+                'total_size_bytes': total_size,
+                'total_size_mb': round(total_size / (1024 * 1024), 2),
+                'max_size_mb': self.max_size_bytes / (1024 * 1024),
+                'hits': self._hits,
+                'misses': self._misses,
+                'hit_rate': round(hit_rate, 1),
+            }
+
+    def fetch_and_cache(self, cache_id: str, url: str) -> Optional[Tuple[bytes, str]]:
+        """Fetch an image from URL and cache it.
+
+        Args:
+            cache_id: Cache key
+            url: URL to fetch from
+
+        Returns:
+            Tuple of (image_data, content_type) or None on failure
+        """
+        try:
+
+            response = requests.get(
+                url,
+                timeout=(5, 10),
+                headers=FETCH_HEADERS,
+                stream=True,
+            )
+            response.raise_for_status()
+
+            # Validate content type
+            content_type = response.headers.get('content-type', '')
+            if not content_type.startswith('image/'):
+                self.put_negative(cache_id)
+                return None
+
+            # Read with size limit
+            data = BytesIO()
+            for chunk in response.iter_content(chunk_size=8192):
+                data.write(chunk)
+                if data.tell() > MAX_IMAGE_SIZE:
+                    self.put_negative(cache_id)
+                    return None
+
+            image_data = data.getvalue()
+
+            if not image_data:
+                self.put_negative(cache_id)
+                return None
+
+            # Store in cache
+            if self.put(cache_id, image_data, content_type):
+                # Get the actual content type from detection
+                detected = _detect_image_type(image_data)
+                if detected:
+                    content_type = detected[0]
+                return image_data, content_type
+
+            return None
+
+        except requests.exceptions.Timeout:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.ConnectionError:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.HTTPError as e:
+            is_404 = e.response is not None and e.response.status_code == 404
+            self.put_negative(cache_id, transient=not is_404)
+            return None
+        except Exception:
+            return None
+
+
+# Singleton instance (initialized lazily when config is available)
+_instance: Optional[ImageCacheService] = None
+_instance_lock = threading.Lock()
+
+
+def get_image_cache() -> ImageCacheService:
+    """Get the singleton image cache instance.
+
+    Lazily initializes using config values.
+    """
+    global _instance
+
+    if _instance is None:
+        with _instance_lock:
+            if _instance is None:
+                from shelfmark.core.config import config
+                from shelfmark.config.env import CONFIG_DIR
+
+                cache_dir = CONFIG_DIR / "covers"
+                max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
+                ttl_days = config.get("COVERS_CACHE_TTL", 0)
+                ttl_seconds = ttl_days * 86400 if ttl_days > 0 else 0
+
+                _instance = ImageCacheService(
+                    cache_dir=cache_dir,
+                    max_size_mb=max_size_mb,
+                    ttl_seconds=ttl_seconds,
+                )
+                logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
+
+    return _instance
+
+
+def reset_image_cache() -> None:
+    """Reset the singleton instance (for testing or config changes)."""
+    global _instance
+    with _instance_lock:
+        _instance = None

shelfmark/core/logger.py

@@ -1,15 +1,17 @@
-"""Centralized logging configuration for the book downloader application."""
+"""Logging configuration and custom logger with error tracing."""
 
 import logging
 import sys
 from pathlib import Path
 from logging.handlers import RotatingFileHandler
-from env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
 from typing import Any
 
+from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
+
+
 class CustomLogger(logging.Logger):
     """Custom logger class with additional error_trace method."""
-    
+
     def error_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log an error message with full stack trace."""
         self.log_resource_usage()
@@ -21,7 +23,7 @@ class CustomLogger(logging.Logger):
         self.log_resource_usage()
         kwargs.pop('exc_info', None)
         self.warning(msg, *args, exc_info=True, **kwargs)
-    
+
     def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log an info message (stack trace only if exception active)."""
         kwargs.pop('exc_info', None)
@@ -35,44 +37,44 @@ class CustomLogger(logging.Logger):
         # Only include exc_info if there's actually an exception
         has_exception = sys.exc_info()[0] is not None
         self.debug(msg, *args, exc_info=has_exception, **kwargs)
-    
+
     def log_resource_usage(self):
         import psutil
+
+        # Sum RSS of all processes for actual app memory
+        app_memory_mb = 0
+        for proc in psutil.process_iter(['memory_info']):
+            try:
+                if proc.info['memory_info']:
+                    app_memory_mb += proc.info['memory_info'].rss / (1024 * 1024)
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+
         memory = psutil.virtual_memory()
+        system_used_mb = memory.used / (1024 * 1024)
         available_mb = memory.available / (1024 * 1024)
-        memory_used_mb = memory.used / (1024 * 1024)
         cpu_percent = psutil.cpu_percent()
-        self.debug(f"Container Memory: Available={available_mb:.2f} MB, Used={memory_used_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
+        self.debug(f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
 
 
 def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     """Set up and configure a logger instance.
-    
+
     Args:
         name: The name of the logger instance
         log_file: Optional path to log file. If None, logs only to stdout/stderr
-        
+
     Returns:
         CustomLogger: Configured logger instance with error_trace method
     """
     # Register our custom logger class
     logging.setLoggerClass(CustomLogger)
-    
+
     # Create logger as CustomLogger instance
     logger = CustomLogger(name)
-    log_level = logging.INFO
-    if LOG_LEVEL == "DEBUG":
-        log_level = logging.DEBUG
-    elif LOG_LEVEL == "INFO":
-        log_level = logging.INFO
-    elif LOG_LEVEL == "WARNING":
-        log_level = logging.WARNING
-    elif LOG_LEVEL == "ERROR":
-        log_level = logging.ERROR
-    elif LOG_LEVEL == "CRITICAL":
-        log_level = logging.CRITICAL
+    log_level = getattr(logging, LOG_LEVEL, logging.INFO)
     logger.setLevel(log_level)
-    
+
     formatter = logging.Formatter(
         '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
     )
@@ -83,13 +85,13 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     console_handler.setLevel(log_level)
     console_handler.addFilter(lambda record: record.levelno < logging.ERROR)  # Only allow logs below ERROR to stdout
     logger.addHandler(console_handler)
-    
+
     # Error handler for stderr
     error_handler = logging.StreamHandler(sys.stderr)
     error_handler.setLevel(logging.ERROR) # Error and above go to stderr
     error_handler.setFormatter(formatter)
     logger.addHandler(error_handler)
-    
+
     # File handler if log file is specified
     try:
         if ENABLE_LOGGING:
@@ -107,4 +109,3 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
         logger.error_trace(f"Failed to create log file: {e}", exc_info=True)
 
     return logger
-

shelfmark/core/mirrors.py

@@ -0,0 +1,231 @@
+"""Centralized mirror configuration for all download sources."""
+
+from typing import List
+
+from shelfmark.core.utils import normalize_http_url
+
+# Lazy import to avoid circular imports
+_config_module = None
+
+
+def _get_config():
+    """Lazy import of config module to avoid circular imports."""
+    global _config_module
+    if _config_module is None:
+        from shelfmark.core.config import config
+        _config_module = config
+    return _config_module
+
+
+# Default mirror lists (hardcoded fallbacks)
+DEFAULT_AA_MIRRORS = [
+    "https://annas-archive.se",
+    "https://annas-archive.li",
+    "https://annas-archive.pm",
+    "https://annas-archive.in",
+]
+
+DEFAULT_LIBGEN_MIRRORS = [
+    "https://libgen.gl",
+    "https://libgen.li",
+    "https://libgen.bz",
+    "https://libgen.la",
+    "https://libgen.vg",
+]
+
+DEFAULT_ZLIB_MIRRORS = [
+    "https://z-lib.fm",
+    "https://z-lib.gs",
+    "https://z-lib.id",
+    "https://z-library.sk",
+    "https://zlibrary-global.se",
+]
+
+DEFAULT_WELIB_MIRRORS = [
+    "https://welib.org",
+]
+
+
+def _normalize_mirror_url(url: str) -> str:
+    return normalize_http_url(url, default_scheme="https")
+
+
+def get_aa_mirrors() -> List[str]:
+    """
+    Get Anna's Archive mirrors from config + defaults.
+
+    Returns:
+        List of AA mirror URLs, starting with defaults then custom additions.
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("AA_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_libgen_mirrors() -> List[str]:
+    """
+    Get LibGen mirrors: defaults + any additional from config.
+
+    Returns:
+        List of LibGen mirror URLs (defaults first, then custom additions).
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_mirrors() -> List[str]:
+    """
+    Get Z-Library mirrors, with primary first.
+
+    Returns:
+        List of Z-Library mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_primary_url() -> str:
+    """
+    Get the primary Z-Library mirror URL.
+
+    Returns:
+        Primary Z-Library mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+
+
+def get_zlib_url_template() -> str:
+    """
+    Get Z-Library URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_zlib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_welib_mirrors() -> List[str]:
+    """
+    Get Welib mirrors, with primary first.
+
+    Returns:
+        List of Welib mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_WELIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("WELIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_welib_primary_url() -> str:
+    """
+    Get the primary Welib mirror URL.
+
+    Returns:
+        Primary Welib mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+
+
+def get_welib_url_template() -> str:
+    """
+    Get Welib URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_welib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_zlib_cookie_domains() -> set:
+    """
+    Get set of Z-Library domains that need full cookie handling.
+
+    Used by internal_bypasser for CF bypass cookie management.
+
+    Returns:
+        Set of domain strings (without protocol).
+    """
+    domains = set()
+
+    # Add all default domains
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized:
+            domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+            domains.add(domain)
+
+    # Add custom domains
+    config = _get_config()
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized:
+                domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+                domains.add(domain)
+
+    return domains

shelfmark/core/models.py

@@ -0,0 +1,170 @@
+"""Data structures and models used across the application."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional
+from enum import Enum
+import re
+import time
+
+
+def build_filename(
+    title: str,
+    author: Optional[str] = None,
+    year: Optional[str] = None,
+    fmt: Optional[str] = None,
+) -> str:
+    parts = []
+    if author:
+        parts.append(author)
+        parts.append(" - ")
+    parts.append(title)
+    if year:
+        parts.append(f" ({year})")
+
+    filename = "".join(parts)
+    filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
+
+    if fmt:
+        filename = f"{filename}.{fmt}"
+
+    return filename
+
+
+class QueueStatus(str, Enum):
+    """Enum for possible book queue statuses."""
+    QUEUED = "queued"
+    RESOLVING = "resolving"
+    DOWNLOADING = "downloading"
+    COMPLETE = "complete"
+    AVAILABLE = "available"
+    ERROR = "error"
+    DONE = "done"
+    CANCELLED = "cancelled"
+
+
+class SearchMode(str, Enum):
+    DIRECT = "direct"
+    UNIVERSAL = "universal"
+
+
+@dataclass
+class QueueItem:
+    """Queue item with priority and metadata."""
+    book_id: str
+    priority: int
+    added_time: float
+
+    def __lt__(self, other):
+        """Compare items for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+
+@dataclass
+class DownloadTask:
+    task_id: str                                # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
+    source: str                                 # Handler name ("direct_download", "prowlarr")
+    title: str                                  # Display title for queue sidebar
+
+    # Display info for queue sidebar
+    author: Optional[str] = None
+    year: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    preview: Optional[str] = None
+    content_type: Optional[str] = None  # "book (fiction)", "audiobook", "magazine", etc.
+
+    # Series info (for library naming templates)
+    series_name: Optional[str] = None
+    series_position: Optional[float] = None  # Float for novellas (e.g., 1.5)
+    subtitle: Optional[str] = None  # Book subtitle for naming templates
+
+    # Hardlinking support
+    original_download_path: Optional[str] = None  # Path in download client (for hardlinking)
+
+    # Search mode - determines post-download processing behavior
+    # See SearchMode enum for behavioral differences
+    search_mode: Optional[SearchMode] = None
+
+    # Runtime state
+    priority: int = 0
+    added_time: float = field(default_factory=time.time)
+    progress: float = 0.0
+    status: QueueStatus = QueueStatus.QUEUED
+    status_message: Optional[str] = None
+    download_path: Optional[str] = None
+
+    def __lt__(self, other):
+        """Compare tasks for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+    def get_filename(self) -> str:
+        """Build sanitized filename from task metadata."""
+        if self.download_path:
+            return Path(self.download_path).name
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class BookInfo:
+    """Data class representing book information."""
+    id: str
+    title: str
+    preview: Optional[str] = None
+    author: Optional[str] = None
+    publisher: Optional[str] = None
+    year: Optional[str] = None
+    language: Optional[str] = None
+    content: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    info: Optional[Dict[str, List[str]]] = None
+    description: Optional[str] = None
+    download_urls: List[str] = field(default_factory=list)
+    download_path: Optional[str] = None
+    priority: int = 0
+    progress: Optional[float] = None
+    status_message: Optional[str] = None  # Detailed status message for UI display
+    added_time: Optional[float] = None  # Timestamp when added to queue
+    source: str = "direct_download"  # Release source handler to use for downloads
+    source_url: Optional[str] = None  # Link to source page (e.g., Anna's Archive)
+
+    def get_filename(self, fallback_url: Optional[str] = None) -> str:
+        """Build sanitized filename: 'Author - Title (Year).format'
+
+        Resolves format from self.format, download_urls, or fallback_url.
+
+        Args:
+            fallback_url: URL to extract format from if not already known
+
+        Returns:
+            Sanitized filename safe for filesystem use
+        """
+        # Resolve format if needed
+        if not self.format:
+            urls = [self.download_urls[0]] if self.download_urls else []
+            if fallback_url:
+                urls.append(fallback_url)
+            for url in urls:
+                ext = url.split(".")[-1].lower()
+                if ext and len(ext) <= 5 and ext.isalnum():
+                    self.format = ext
+                    break
+
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class SearchFilters:
+    """Filters for book search queries."""
+    isbn: Optional[List[str]] = None
+    author: Optional[List[str]] = None
+    title: Optional[List[str]] = None
+    lang: Optional[List[str]] = None
+    sort: Optional[str] = None
+    content: Optional[List[str]] = None
+    format: Optional[List[str]] = None

shelfmark/core/naming.py

@@ -0,0 +1,201 @@
+"""Template-based naming for library organization."""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional, Union, Mapping
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+TOKEN_PATTERN = re.compile(
+    r'\{([- ._/\[(]*)'   # prefix: space, dash, dot, underscore, slash, brackets
+    r'([A-Za-z]+)'        # token name
+    r'([- ._/\])]*)\}'    # suffix: space, dash, dot, underscore, slash, brackets
+)
+
+# Characters that are invalid in filenames on various filesystems
+INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def _sanitize(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for filesystem use."""
+    if not name:
+        return ""
+
+    sanitized = INVALID_CHARS.sub('_', name)
+    sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized)  # Strip whitespace and dots
+    sanitized = re.sub(r'_+', '_', sanitized)  # Collapse underscores
+    return sanitized[:max_length]
+
+
+def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for use as a filename or path component."""
+    return _sanitize(name, max_length)
+
+
+# Alias for backwards compatibility
+sanitize_path_component = sanitize_filename
+
+
+def format_series_position(position: Optional[Union[str, int, float]]) -> str:
+    if position is None:
+        return ""
+
+    # Display as integer if whole number
+    if isinstance(position, float) and position.is_integer():
+        return str(int(position))
+
+    return str(position)
+
+
+# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
+PAD_NUMBERS_PATTERN = re.compile(r'\d+')
+
+
+def natural_sort_key(path: Union[str, Path]) -> str:
+    """Generate a sort key with padded numbers for natural sorting."""
+    filename = Path(path).name.lower()
+    return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
+
+
+def assign_part_numbers(
+    files: list[Path],
+    zero_pad_width: int = 2,
+) -> list[tuple[Path, str]]:
+    """Sort files naturally and assign sequential part numbers (1, 2, 3...)."""
+    if not files:
+        return []
+
+    sorted_files = sorted(files, key=natural_sort_key)
+    return [
+        (file_path, str(part_num).zfill(zero_pad_width))
+        for part_num, file_path in enumerate(sorted_files, start=1)
+    ]
+
+
+def parse_naming_template(
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    *,
+    allow_path_separators: bool = True,
+) -> str:
+    if not template:
+        return ""
+
+    # Normalize metadata keys to lowercase for case-insensitive matching
+    normalized = {k.lower(): v for k, v in metadata.items()}
+
+    def replace_token(match: re.Match) -> str:
+        prefix = match.group(1)
+        token_name = match.group(2).lower()
+        suffix = match.group(3)
+
+        # Get the value for this token
+        value = normalized.get(token_name)
+
+        # Special handling for series position
+        if token_name == 'seriesposition':
+            value = format_series_position(value)
+
+        # Convert to string
+        if value is None:
+            value = ""
+        else:
+            value = str(value).strip()
+
+        # If value is empty, return empty string (no prefix/suffix)
+        if not value:
+            return ""
+
+        if not allow_path_separators:
+            value = value.replace("/", "_")
+        # Sanitize the value
+        value = sanitize_filename(value)
+
+        return f"{prefix}{value}{suffix}"
+
+    # Replace all tokens
+    result = TOKEN_PATTERN.sub(replace_token, template)
+
+    # Clean up any double slashes that might result from empty tokens
+    result = re.sub(r'/+', '/', result)
+
+    # Remove leading/trailing slashes
+    result = result.strip('/')
+
+    # Clean up any orphaned separators (e.g., " - " at start/end, or " -  - ")
+    result = re.sub(r'^[\s\-_.]+', '', result)
+    result = re.sub(r'[\s\-_.]+$', '', result)
+    result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
+
+    # Clean up empty parentheses/brackets
+    result = re.sub(r'\(\s*\)', '', result)
+    result = re.sub(r'\[\s*\]', '', result)
+
+    # Final trim of any trailing separators left after cleanup
+    result = re.sub(r'[\s\-_.]+$', '', result)
+
+    return result
+
+
+def build_library_path(
+    base_path: str,
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    extension: Optional[str] = None,
+) -> Path:
+    relative = parse_naming_template(template, metadata, allow_path_separators=True)
+
+    if not relative:
+        # Fallback to title if template produces empty result
+        title = metadata.get('Title') or metadata.get('title') or 'Unknown'
+        relative = sanitize_filename(str(title))
+
+    # Remove any path traversal attempts
+    relative = relative.replace('..', '')
+
+    base = Path(base_path).resolve()
+    full_path = (base / relative).resolve()
+
+    # Verify the path is within the base directory
+    try:
+        full_path.relative_to(base)
+    except ValueError:
+        raise ValueError(f"Path traversal detected: template would escape library directory")
+
+    if extension:
+        ext = extension.lstrip('.')
+        # Don't use with_suffix() - it replaces everything after the first dot
+        # e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
+        full_path = Path(f"{full_path}.{ext}")
+
+    return full_path
+
+
+def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
+    """Check if two paths are on the same filesystem."""
+    path1 = Path(path1)
+    path2 = Path(path2)
+
+    def get_device(p: Path) -> Optional[int]:
+        try:
+            while not p.exists():
+                p = p.parent
+                if p == p.parent:
+                    break
+            return os.stat(p).st_dev
+        except (OSError, PermissionError) as e:
+            logger.debug(f"Cannot stat {p}: {e}")
+            return None
+
+    dev1 = get_device(path1)
+    dev2 = get_device(path2)
+
+    if dev1 is None or dev2 is None:
+        logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
+        return False
+
+    return dev1 == dev2

shelfmark/core/onboarding.py

@@ -0,0 +1,429 @@
+"""
+Onboarding wizard configuration.
+
+Defines the steps and fields for the first-run onboarding experience.
+Reuses field definitions from the settings registry where possible.
+"""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    HeadingField,
+    SettingsField,
+    get_settings_tab,
+    serialize_field,
+    save_config_file,
+    get_setting_value,
+)
+
+logger = setup_logger(__name__)
+
+
+ONBOARDING_STORAGE_KEY = "onboarding_complete"
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def is_onboarding_complete() -> bool:
+    """Check if onboarding has been completed."""
+    config_file = _get_config_dir() / "settings.json"
+    if not config_file.exists():
+        return False
+
+    try:
+        with open(config_file, 'r') as f:
+            config = json.load(f)
+            return config.get(ONBOARDING_STORAGE_KEY, False)
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning(f"Could not read onboarding status from settings.json: {e}")
+        return False
+
+
+def mark_onboarding_complete() -> bool:
+    """Mark onboarding as complete."""
+    try:
+        return save_config_file("general", {ONBOARDING_STORAGE_KEY: True})
+    except Exception as e:
+        logger.error(f"Failed to mark onboarding complete: {e}")
+        return False
+
+
+def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField]:
+    """
+    Extract a specific field from a registered settings tab.
+
+    Args:
+        tab_name: Name of the settings tab (e.g., 'search_mode', 'hardcover')
+        field_key: Key of the field to extract (e.g., 'SEARCH_MODE', 'HARDCOVER_API_KEY')
+
+    Returns:
+        The field if found, None otherwise
+    """
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        logger.warning(f"Settings tab not found: {tab_name}")
+        return None
+
+    for field in tab.fields:
+        if hasattr(field, 'key') and field.key == field_key:
+            return field
+
+    logger.warning(f"Field {field_key} not found in tab {tab_name}")
+    return None
+
+
+def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
+    """
+    Clone a field with optional attribute overrides.
+
+    Useful for customizing labels, descriptions, or defaults for onboarding context.
+    """
+    return replace(field, **overrides)
+
+
+# =============================================================================
+# Step Definitions
+# =============================================================================
+
+
+def get_search_mode_fields() -> List[SettingsField]:
+    """Step 1: Choose search mode - uses actual SEARCH_MODE field from settings."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="welcome_heading",
+            title="Welcome to Shelfmark",
+            description="Let's configure how you want to search for and download books.",
+        ),
+    ]
+
+    # Get the actual SEARCH_MODE field from settings
+    search_mode_field = _get_field_from_tab("search_mode", "SEARCH_MODE")
+    if search_mode_field:
+        # Clone with onboarding-specific description
+        fields.append(_clone_field_with_overrides(
+            search_mode_field,
+            description="Choose how you want to find books.",
+        ))
+
+    return fields
+
+
+def get_metadata_provider_fields() -> List[SettingsField]:
+    """Step 2: Choose metadata provider - uses actual METADATA_PROVIDER field."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="metadata_heading",
+            title="Metadata Provider",
+            description="Choose where to search for book information. You can enable more providers in Settings later.",
+        ),
+    ]
+
+    # Get the actual METADATA_PROVIDER field from settings
+    provider_field = _get_field_from_tab("search_mode", "METADATA_PROVIDER")
+    if provider_field:
+        # Custom options with Hardcover marked as recommended
+        onboarding_options = [
+            {
+                "value": "hardcover",
+                "label": "Hardcover (Recommended)",
+                "description": "Modern book tracking platform with excellent metadata, ratings, and series information. Requires free API key.",
+            },
+            {
+                "value": "openlibrary",
+                "label": "Open Library",
+                "description": "Free, open-source library catalog from the Internet Archive. No API key required.",
+            },
+            {
+                "value": "googlebooks",
+                "label": "Google Books",
+                "description": "Google's book database with good coverage. Requires free API key.",
+            },
+        ]
+
+        # Clone with onboarding-specific options and default
+        fields.append(_clone_field_with_overrides(
+            provider_field,
+            default="hardcover",
+            options=onboarding_options,
+        ))
+
+    return fields
+
+
+def get_hardcover_setup_fields() -> List[SettingsField]:
+    """Step 3a: Configure Hardcover - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="hardcover_setup_heading",
+            title="Hardcover Setup",
+            description="Get your free API key from hardcover.app/account/api",
+            link_url="https://hardcover.app/account/api",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual HARDCOVER_API_KEY field
+    api_key_field = _get_field_from_tab("hardcover", "HARDCOVER_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("hardcover", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_googlebooks_setup_fields() -> List[SettingsField]:
+    """Step 3b: Configure Google Books - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="googlebooks_setup_heading",
+            title="Google Books Setup",
+            description="Get your free API key from Google Cloud Console (APIs & Services > Credentials).",
+            link_url="https://console.cloud.google.com/apis/library/books.googleapis.com",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual GOOGLEBOOKS_API_KEY field
+    api_key_field = _get_field_from_tab("googlebooks", "GOOGLEBOOKS_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("googlebooks", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_prowlarr_fields() -> List[SettingsField]:
+    """Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_heading",
+            title="Prowlarr Integration (Optional)",
+            description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.",
+        ),
+    ]
+
+    # Get actual Prowlarr connection fields
+    prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"]
+    for field_key in prowlarr_fields:
+        field = _get_field_from_tab("prowlarr_config", field_key)
+        if field:
+            fields.append(field)
+
+    return fields
+
+
+def get_prowlarr_indexers_fields() -> List[SettingsField]:
+    """Step 5: Select Prowlarr indexers to search."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_indexers_heading",
+            title="Select Indexers",
+            description="Choose which indexers to search for books. Leave empty to search all available indexers.",
+        ),
+    ]
+
+    # Get the indexers multi-select field
+    indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS")
+    if indexers_field:
+        fields.append(indexers_field)
+
+    return fields
+
+
+# =============================================================================
+# Step Configuration
+# =============================================================================
+
+
+ONBOARDING_STEPS = [
+    {
+        "id": "search_mode",
+        "title": "Search Mode",
+        "tab": "search_mode",
+        "get_fields": get_search_mode_fields,
+    },
+    {
+        "id": "metadata_provider",
+        "title": "Metadata Provider",
+        "tab": "search_mode",
+        "get_fields": get_metadata_provider_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+    },
+    {
+        "id": "hardcover_setup",
+        "title": "Hardcover Setup",
+        "tab": "hardcover",
+        "get_fields": get_hardcover_setup_fields,
+        # Must be universal mode AND hardcover selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "hardcover"},
+        ],
+    },
+    {
+        "id": "googlebooks_setup",
+        "title": "Google Books Setup",
+        "tab": "googlebooks",
+        "get_fields": get_googlebooks_setup_fields,
+        # Must be universal mode AND googlebooks selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "googlebooks"},
+        ],
+    },
+    {
+        "id": "prowlarr",
+        "title": "Prowlarr",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+        "optional": True,
+    },
+    {
+        "id": "prowlarr_indexers",
+        "title": "Indexers",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_indexers_fields,
+        # Only show when Prowlarr is enabled
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "PROWLARR_ENABLED", "value": True},
+        ],
+        "optional": True,
+    },
+]
+
+
+def get_onboarding_config() -> Dict[str, Any]:
+    """
+    Get the full onboarding configuration including steps and current values.
+    """
+    steps = []
+    all_values = {}
+
+    for step_config in ONBOARDING_STEPS:
+        fields = step_config["get_fields"]()
+        tab_name = step_config["tab"]
+
+        # Serialize fields with current values
+        serialized_fields = []
+        for field in fields:
+            serialized = serialize_field(field, tab_name, include_value=True)
+            serialized_fields.append(serialized)
+
+            # Collect values (skip HeadingFields)
+            if hasattr(field, 'key') and field.key and not isinstance(field, HeadingField):
+                value = get_setting_value(field, tab_name)
+                all_values[field.key] = value if value is not None else getattr(field, 'default', '')
+
+        step = {
+            "id": step_config["id"],
+            "title": step_config["title"],
+            "tab": tab_name,
+            "fields": serialized_fields,
+        }
+
+        if "show_when" in step_config:
+            step["showWhen"] = step_config["show_when"]
+        if step_config.get("optional"):
+            step["optional"] = True
+
+        steps.append(step)
+
+    return {
+        "steps": steps,
+        "values": all_values,
+        "complete": is_onboarding_complete(),
+    }
+
+
+def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Save onboarding settings and mark as complete.
+
+    Args:
+        values: Dict of field key -> value
+
+    Returns:
+        Dict with success status and message
+    """
+    try:
+        # Group values by their target tab
+        tab_values: Dict[str, Dict[str, Any]] = {}
+
+        for step_config in ONBOARDING_STEPS:
+            tab_name = step_config["tab"]
+            fields = step_config["get_fields"]()
+
+            for field in fields:
+                if isinstance(field, HeadingField):
+                    continue
+
+                key = field.key
+                if key in values:
+                    if tab_name not in tab_values:
+                        tab_values[tab_name] = {}
+                    tab_values[tab_name][key] = values[key]
+
+        # Save each tab's values
+        for tab_name, tab_data in tab_values.items():
+            if tab_data:
+                save_config_file(tab_name, tab_data)
+                logger.info(f"Saved onboarding settings to {tab_name}: {list(tab_data.keys())}")
+
+        # Enable the selected metadata provider
+        search_mode = values.get("SEARCH_MODE", "direct")
+        if search_mode == "universal":
+            provider = values.get("METADATA_PROVIDER", "hardcover")
+            if provider:
+                # Map provider name to its enabled key
+                enabled_key_map = {
+                    "hardcover": "HARDCOVER_ENABLED",
+                    "openlibrary": "OPENLIBRARY_ENABLED",
+                    "googlebooks": "GOOGLEBOOKS_ENABLED",
+                }
+                enabled_key = enabled_key_map.get(provider, f"{provider.upper()}_ENABLED")
+
+                # Get existing provider config and add enabled flag
+                provider_config = {enabled_key: True}
+
+                # Include API key if provided for that provider
+                if provider == "hardcover" and values.get("HARDCOVER_API_KEY"):
+                    provider_config["HARDCOVER_API_KEY"] = values["HARDCOVER_API_KEY"]
+                elif provider == "googlebooks" and values.get("GOOGLEBOOKS_API_KEY"):
+                    provider_config["GOOGLEBOOKS_API_KEY"] = values["GOOGLEBOOKS_API_KEY"]
+
+                save_config_file(provider, provider_config)
+                logger.info(f"Enabled metadata provider: {provider} with keys: {list(provider_config.keys())}")
+
+        # Mark onboarding as complete
+        mark_onboarding_complete()
+
+        # Refresh config
+        try:
+            from shelfmark.core.config import config
+            config.refresh()
+        except ImportError as e:
+            logger.debug(f"Could not refresh config after onboarding: {e}")
+
+        return {"success": True, "message": "Onboarding complete!"}
+
+    except Exception as e:
+        logger.error(f"Failed to save onboarding settings: {e}")
+        return {"success": False, "message": str(e)}

shelfmark/core/path_mappings.py

@@ -0,0 +1,136 @@
+"""Remote path mapping utilities.
+
+Used when an external download client reports a completed download path that does
+not exist inside the Shelfmark runtime environment (commonly different Docker
+volume mounts).
+
+A mapping rewrites a remote path prefix into a local path prefix.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+
+@dataclass(frozen=True)
+class RemotePathMapping:
+    host: str
+    remote_path: str
+    local_path: str
+
+
+def _normalize_prefix(path: str) -> str:
+    normalized = str(path or "").strip()
+    if not normalized:
+        return ""
+
+    normalized = normalized.replace("\\", "/")
+
+    if normalized != "/":
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def _is_windows_path(path: str) -> bool:
+    """Check if a path looks like a Windows path (has a drive letter like C:/)."""
+    return len(path) >= 2 and path[1] == ":" and path[0].isalpha()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host or "").strip().lower()
+
+
+def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
+    if not value or not isinstance(value, list):
+        return []
+
+    mappings: list[RemotePathMapping] = []
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        host = _normalize_host(row.get("host", ""))
+        remote_path = _normalize_prefix(row.get("remotePath", ""))
+        local_path = _normalize_prefix(row.get("localPath", ""))
+
+        if not host or not remote_path or not local_path:
+            continue
+
+        mappings.append(RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path))
+
+    mappings.sort(key=lambda m: len(m.remote_path), reverse=True)
+    return mappings
+
+
+def remap_remote_to_local_with_match(
+    *,
+    mappings: Iterable[RemotePathMapping],
+    host: str,
+    remote_path: str | Path,
+) -> tuple[Path, bool]:
+    host_normalized = _normalize_host(host)
+    remote_normalized = _normalize_prefix(str(remote_path))
+
+    if not remote_normalized:
+        return Path(str(remote_path)), False
+
+    # Windows paths are case-insensitive, so we need case-insensitive matching
+    # for paths that look like Windows paths (e.g., D:/Torrents)
+    is_windows = _is_windows_path(remote_normalized)
+
+    for mapping in mappings:
+        if _normalize_host(mapping.host) != host_normalized:
+            continue
+
+        remote_prefix = _normalize_prefix(mapping.remote_path)
+        if not remote_prefix:
+            continue
+
+        # For Windows paths, do case-insensitive prefix matching
+        if is_windows:
+            remote_lower = remote_normalized.lower()
+            prefix_lower = remote_prefix.lower()
+            matches = remote_lower == prefix_lower or remote_lower.startswith(prefix_lower + "/")
+        else:
+            matches = remote_normalized == remote_prefix or remote_normalized.startswith(remote_prefix + "/")
+
+        if matches:
+            # Use the length of the original prefix to extract remainder
+            # This preserves the original case in folder names
+            remainder = remote_normalized[len(remote_prefix):]
+            local_prefix = _normalize_prefix(mapping.local_path)
+
+            if remainder.startswith("/"):
+                remainder = remainder[1:]
+
+            remapped = Path(local_prefix) / remainder if remainder else Path(local_prefix)
+            return remapped, True
+
+    return Path(remote_normalized), False
+
+
+def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path) -> Path:
+    remapped, _ = remap_remote_to_local_with_match(
+        mappings=mappings,
+        host=host,
+        remote_path=remote_path,
+    )
+    return remapped
+
+
+def get_client_host_identifier(client: Any) -> Optional[str]:
+    """Return a stable identifier used by the mapping UI.
+
+    Sonarr uses the download client's configured host. Shelfmark currently uses
+    the download client 'name' (e.g. qbittorrent, sabnzbd).
+    """
+
+    name = getattr(client, "name", None)
+    if isinstance(name, str) and name.strip():
+        return name.strip().lower()
+
+    return None

shelfmark/core/prefix_middleware.py

@@ -0,0 +1,31 @@
+"""WSGI middleware for hosting Shelfmark under a URL prefix."""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+
+class PrefixMiddleware:
+    """Strip a configured URL prefix from PATH_INFO before routing."""
+
+    def __init__(self, app, prefix: str, bypass_paths: Optional[Iterable[str]] = None) -> None:
+        self.app = app
+        self.prefix = prefix.rstrip("/")
+        self.bypass_paths = set(bypass_paths or [])
+
+    def __call__(self, environ, start_response):
+        path = environ.get("PATH_INFO", "") or ""
+
+        if path in self.bypass_paths:
+            return self.app(environ, start_response)
+
+        if not self.prefix:
+            return self.app(environ, start_response)
+
+        if path == self.prefix or path.startswith(self.prefix + "/"):
+            environ["SCRIPT_NAME"] = self.prefix
+            environ["PATH_INFO"] = path[len(self.prefix):] or "/"
+            return self.app(environ, start_response)
+
+        start_response("404 Not Found", [("Content-Type", "text/plain")])
+        return [b"Not Found"]

shelfmark/core/queue.py

@@ -0,0 +1,296 @@
+"""Thread-safe download queue manager with priority support and cancellation."""
+
+import queue
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from threading import Lock, Event
+from typing import Dict, List, Optional, Tuple, Any
+
+from shelfmark.core.config import config as app_config
+from shelfmark.core.models import QueueStatus, QueueItem, DownloadTask
+
+
+class BookQueue:
+    """Thread-safe download queue manager with priority support and cancellation."""
+
+    def __init__(self) -> None:
+        self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
+        self._lock = Lock()
+        self._status: dict[str, QueueStatus] = {}
+        self._task_data: dict[str, DownloadTask] = {}
+        self._status_timestamps: dict[str, datetime] = {}  # Track when each status was last updated
+        self._cancel_flags: dict[str, Event] = {}  # Cancellation flags for active downloads
+        self._active_downloads: dict[str, bool] = {}  # Track currently downloading tasks
+
+    @property
+    def _status_timeout(self) -> timedelta:
+        """Get status timeout from config (allows live updates)."""
+        return timedelta(seconds=app_config.get("STATUS_TIMEOUT", 3600))
+
+    def add(self, task: DownloadTask) -> bool:
+        """Add a download task to the queue. Returns False if already exists."""
+        with self._lock:
+            task_id = task.task_id
+
+            # Don't add if already exists and not in error/done state
+            if task_id in self._status and self._status[task_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
+                return False
+
+            # Ensure added_time is set
+            if task.added_time == 0:
+                task.added_time = time.time()
+
+            queue_item = QueueItem(task_id, task.priority, task.added_time)
+            self._queue.put(queue_item)
+            self._task_data[task_id] = task
+            self._update_status(task_id, QueueStatus.QUEUED)
+            return True
+
+    def get_next(self) -> Optional[Tuple[str, Event]]:
+        """Get next task ID from queue with cancellation flag."""
+        # Use iterative approach to avoid stack overflow if many items are cancelled
+        while True:
+            try:
+                queue_item = self._queue.get_nowait()
+                task_id = queue_item.book_id  # QueueItem uses book_id as the ID field
+
+                with self._lock:
+                    # Check if task was cancelled while in queue
+                    if task_id in self._status and self._status[task_id] == QueueStatus.CANCELLED:
+                        continue  # Skip cancelled items, try next
+
+                    # Create cancellation flag for this download
+                    cancel_flag = Event()
+                    self._cancel_flags[task_id] = cancel_flag
+                    self._active_downloads[task_id] = True
+
+                return task_id, cancel_flag
+            except queue.Empty:
+                return None
+
+    def get_task(self, task_id: str) -> Optional[DownloadTask]:
+        """Get a task by its ID."""
+        with self._lock:
+            return self._task_data.get(task_id)
+
+    def _update_status(self, book_id: str, status: QueueStatus) -> None:
+        """Internal method to update status and timestamp."""
+        self._status[book_id] = status
+        self._status_timestamps[book_id] = datetime.now()
+
+    def update_status(self, book_id: str, status: QueueStatus) -> None:
+        """Update status of a book in the queue."""
+        with self._lock:
+            self._update_status(book_id, status)
+
+            # Clean up active download tracking when finished
+            if status in [QueueStatus.COMPLETE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
+                self._active_downloads.pop(book_id, None)
+                self._cancel_flags.pop(book_id, None)
+
+    def update_download_path(self, task_id: str, download_path: str) -> None:
+        """Update the download path of a task in the queue."""
+        with self._lock:
+            if task_id in self._task_data:
+                self._task_data[task_id].download_path = download_path
+
+    def update_progress(self, task_id: str, progress: float) -> None:
+        """Update download progress for a task."""
+        with self._lock:
+            if task_id in self._task_data:
+                self._task_data[task_id].progress = progress
+
+    def update_status_message(self, task_id: str, message: str) -> None:
+        """Update detailed status message for a task."""
+        with self._lock:
+            if task_id in self._task_data:
+                self._task_data[task_id].status_message = message
+
+    def get_status(self) -> Dict[QueueStatus, Dict[str, DownloadTask]]:
+        """Get current queue status grouped by status."""
+        self.refresh()
+        with self._lock:
+            result: Dict[QueueStatus, Dict[str, DownloadTask]] = {status: {} for status in QueueStatus}
+            for task_id, status in self._status.items():
+                if task_id in self._task_data:
+                    result[status][task_id] = self._task_data[task_id]
+            return result
+
+    def get_queue_order(self) -> List[Dict[str, Any]]:
+        """Get current queue order for display."""
+        with self._lock:
+            queue_items = []
+
+            # Get items from priority queue without removing them
+            temp_items = []
+            while not self._queue.empty():
+                try:
+                    item = self._queue.get_nowait()
+                    temp_items.append(item)
+                    task_id = item.book_id  # QueueItem uses book_id as the ID field
+                    if task_id in self._task_data:
+                        task = self._task_data[task_id]
+                        queue_items.append({
+                            'id': task_id,
+                            'title': task.title,
+                            'author': task.author,
+                            'priority': item.priority,
+                            'added_time': item.added_time,
+                            'status': self._status.get(task_id, QueueStatus.QUEUED)
+                        })
+                except queue.Empty:
+                    break
+
+            # Put items back in queue
+            for item in temp_items:
+                self._queue.put(item)
+
+            return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
+
+    def cancel_download(self, task_id: str) -> bool:
+        """Cancel a download or clear a completed/errored item."""
+        with self._lock:
+            current_status = self._status.get(task_id)
+
+            # Allow cancellation during any active state
+            if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]:
+                # Signal active download to stop
+                if task_id in self._cancel_flags:
+                    self._cancel_flags[task_id].set()
+                self._update_status(task_id, QueueStatus.CANCELLED)
+                return True
+            elif current_status == QueueStatus.QUEUED:
+                # Remove from queue and mark as cancelled
+                self._update_status(task_id, QueueStatus.CANCELLED)
+                return True
+            elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
+                # Clear completed/errored/cancelled items from tracking
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
+                self._cancel_flags.pop(task_id, None)
+                self._active_downloads.pop(task_id, None)
+                return True
+
+            return False
+
+    def set_priority(self, task_id: str, new_priority: int) -> bool:
+        """Change the priority of a queued task (lower = higher priority)."""
+        with self._lock:
+            if task_id not in self._status or self._status[task_id] != QueueStatus.QUEUED:
+                return False
+
+            # Remove task from queue and re-add with new priority
+            temp_items = []
+            found = False
+
+            while not self._queue.empty():
+                try:
+                    item = self._queue.get_nowait()
+                    if item.book_id == task_id:  # QueueItem uses book_id as the ID field
+                        # Create new item with updated priority
+                        new_item = QueueItem(task_id, new_priority, item.added_time)
+                        temp_items.append(new_item)
+                        found = True
+                        # Update task data priority
+                        if task_id in self._task_data:
+                            self._task_data[task_id].priority = new_priority
+                    else:
+                        temp_items.append(item)
+                except queue.Empty:
+                    break
+
+            # Put all items back
+            for item in temp_items:
+                self._queue.put(item)
+
+            return found
+
+    def reorder_queue(self, task_priorities: Dict[str, int]) -> bool:
+        """Bulk reorder queue by mapping task_id to new priority."""
+        with self._lock:
+            # Extract all items from queue
+            all_items = []
+            while not self._queue.empty():
+                try:
+                    item = self._queue.get_nowait()
+                    task_id = item.book_id  # QueueItem uses book_id as the ID field
+                    # Update priority if specified
+                    if task_id in task_priorities:
+                        new_priority = task_priorities[task_id]
+                        item = QueueItem(task_id, new_priority, item.added_time)
+                        # Update task data priority
+                        if task_id in self._task_data:
+                            self._task_data[task_id].priority = new_priority
+                    all_items.append(item)
+                except queue.Empty:
+                    break
+
+            # Put all items back with updated priorities
+            for item in all_items:
+                self._queue.put(item)
+
+            return True
+
+    def get_active_downloads(self) -> List[str]:
+        """Get list of currently active download task IDs."""
+        with self._lock:
+            return list(self._active_downloads.keys())
+
+    def has_pending_work(self) -> bool:
+        """Check if there are any active downloads or queued items."""
+        with self._lock:
+            if self._active_downloads:
+                return True
+            return any(status == QueueStatus.QUEUED for status in self._status.values())
+
+    def clear_completed(self) -> int:
+        """Remove all completed, errored, or cancelled tasks from tracking."""
+        terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED}
+        with self._lock:
+            to_remove = [task_id for task_id, status in self._status.items() if status in terminal_statuses]
+
+            for task_id in to_remove:
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
+                self._cancel_flags.pop(task_id, None)
+                self._active_downloads.pop(task_id, None)
+
+            return len(to_remove)
+
+    def refresh(self) -> None:
+        """Remove any tasks that are done downloading or have stale status."""
+        terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED}
+        with self._lock:
+            current_time = datetime.now()
+            to_remove = []
+
+            for task_id, status in self._status.items():
+                task = self._task_data.get(task_id)
+                if not task:
+                    continue
+
+                # Clear stale download paths
+                if task.download_path and not Path(task.download_path).exists():
+                    task.download_path = None
+
+                # Mark available downloads as done if file is gone
+                if status == QueueStatus.AVAILABLE and not task.download_path:
+                    self._update_status(task_id, QueueStatus.DONE)
+
+                # Check for stale status entries
+                last_update = self._status_timestamps.get(task_id)
+                if last_update and (current_time - last_update) > self._status_timeout:
+                    if status in terminal_statuses:
+                        to_remove.append(task_id)
+
+            # Remove stale entries
+            for task_id in to_remove:
+                self._status.pop(task_id, None)
+                self._status_timestamps.pop(task_id, None)
+                self._task_data.pop(task_id, None)
+
+# Global instance of BookQueue
+book_queue = BookQueue()

shelfmark/core/search_plan.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional
+
+MANUAL_QUERY_MAX_LEN = 256
+
+from shelfmark.core.config import config
+from shelfmark.metadata_providers import (
+    BookMetadata,
+    group_languages_by_localized_title,
+    build_localized_search_titles,
+)
+
+
+@dataclass(frozen=True)
+class ReleaseSearchVariant:
+    """A single search variant (title + author) associated with languages."""
+
+    title: str
+    author: str
+    languages: Optional[List[str]] = None
+
+    @property
+    def query(self) -> str:
+        return " ".join(part for part in [self.title, self.author] if part).strip()
+
+
+@dataclass(frozen=True)
+class ReleaseSearchPlan:
+    """Pre-computed search inputs shared across release sources."""
+
+    languages: Optional[List[str]]
+    isbn_candidates: List[str]
+    author: str
+    title_variants: List[ReleaseSearchVariant]
+    grouped_title_variants: List[ReleaseSearchVariant]
+    manual_query: Optional[str] = None
+
+    @property
+    def primary_query(self) -> str:
+        return self.title_variants[0].query if self.title_variants else ""
+
+
+def _normalize_languages(languages: Optional[List[str]]) -> Optional[List[str]]:
+    if not languages:
+        default = config.BOOK_LANGUAGE
+        if not default:
+            return None
+        return [str(lang).strip() for lang in default if str(lang).strip()]
+
+    normalized: List[str] = []
+    for lang in languages:
+        if not lang:
+            continue
+        s = str(lang).strip()
+        if not s:
+            continue
+        normalized.append(s)
+
+    if any(lang.lower() == "all" for lang in normalized):
+        return None
+
+    return normalized or None
+
+
+def _pick_search_author(book: BookMetadata) -> str:
+    if book.search_author:
+        return book.search_author
+
+    if not book.authors:
+        return ""
+
+    first = book.authors[0]
+    if "," in first:
+        first = first.split(",")[0].strip()
+
+    return first
+
+
+def _pick_search_title(book: BookMetadata) -> str:
+    return book.search_title or book.title
+
+
+def build_release_search_plan(
+    book: BookMetadata,
+    languages: Optional[List[str]] = None,
+    manual_query: Optional[str] = None,
+) -> ReleaseSearchPlan:
+    resolved_languages = _normalize_languages(languages)
+
+    resolved_manual_query = None
+    if manual_query:
+        resolved_manual_query = manual_query.strip()[:MANUAL_QUERY_MAX_LEN] or None
+
+    author = _pick_search_author(book)
+    base_title = _pick_search_title(book)
+
+    if resolved_manual_query:
+        # Manual override: use the raw query as-is (no language/title expansion).
+        variant = ReleaseSearchVariant(title=resolved_manual_query, author="", languages=None)
+        return ReleaseSearchPlan(
+            languages=resolved_languages,
+            isbn_candidates=[],
+            author="",
+            title_variants=[variant],
+            grouped_title_variants=[variant],
+            manual_query=resolved_manual_query,
+        )
+
+    isbn_candidates: List[str] = []
+    if book.isbn_13:
+        isbn_candidates.append(book.isbn_13)
+    if book.isbn_10 and book.isbn_10 not in isbn_candidates:
+        isbn_candidates.append(book.isbn_10)
+
+    titles_by_language = book.titles_by_language or None
+    if book.search_title and titles_by_language:
+        titles_by_language = {
+            k: v
+            for k, v in titles_by_language.items()
+            if str(k).strip().lower() not in {"en", "eng", "english"}
+        }
+
+    grouped = group_languages_by_localized_title(
+        base_title=base_title,
+        languages=resolved_languages,
+        titles_by_language=titles_by_language,
+    )
+
+    grouped_variants: List[ReleaseSearchVariant] = [
+        ReleaseSearchVariant(title=title, author=author, languages=langs)
+        for title, langs in grouped
+        if title
+    ]
+
+    expanded_titles = build_localized_search_titles(
+        base_title=base_title,
+        languages=resolved_languages,
+        titles_by_language=titles_by_language,
+        excluded_languages={"en", "eng", "english"},
+    )
+
+    title_variants: List[ReleaseSearchVariant] = [
+        ReleaseSearchVariant(title=title, author=author, languages=None)
+        for title in expanded_titles
+        if title
+    ]
+
+    # If no titles could be built, fall back to ISBN queries.
+    if not title_variants and isbn_candidates:
+        title_variants = [
+            ReleaseSearchVariant(title=isbn, author="", languages=None)
+            for isbn in isbn_candidates
+        ]
+
+    return ReleaseSearchPlan(
+        languages=resolved_languages,
+        isbn_candidates=isbn_candidates,
+        author=author,
+        title_variants=title_variants,
+        grouped_title_variants=grouped_variants,
+        manual_query=None,
+    )

shelfmark/core/settings_registry.py

@@ -0,0 +1,906 @@
+"""Plugin settings registry with config file persistence."""
+
+import json
+import os
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Type, Union
+from threading import Lock
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@dataclass
+class FieldBase:
+    """Base class for all settings fields."""
+    key: str                              # Environment variable / config key
+    label: str                            # Display label in UI
+    description: str = ""                 # Help text
+    default: Any = None                   # Default value if not set
+    required: bool = False                # Whether field must have a value
+    env_var: Optional[str] = None         # Override env var name (defaults to key)
+    env_supported: bool = True            # Whether this setting can be set via ENV var (False = UI-only)
+    disabled: bool = False                # Whether field is disabled/greyed out
+    disabled_reason: str = ""             # Explanation shown when disabled
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    disabled_when: Optional[Dict[str, Any]] = None  # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
+    requires_restart: bool = False        # Whether changing this setting requires a container restart
+    universal_only: bool = False          # Only show in Universal search mode (hide in Direct mode)
+
+    def get_env_var_name(self) -> str:
+        """Get the environment variable name for this field."""
+        return self.env_var or self.key
+
+    def get_field_type(self) -> str:
+        """Get the field type name for serialization."""
+        return self.__class__.__name__
+
+
+@dataclass
+class TextField(FieldBase):
+    """Single-line text input."""
+    placeholder: str = ""
+    max_length: Optional[int] = None
+
+
+@dataclass
+class PasswordField(FieldBase):
+    """Password input (masked in UI, not returned in API responses)."""
+    placeholder: str = ""
+
+
+@dataclass
+class NumberField(FieldBase):
+    """Numeric input."""
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+    step: float = 1
+    default: float = 0
+
+
+@dataclass
+class CheckboxField(FieldBase):
+    """Boolean checkbox."""
+    default: bool = False
+
+
+@dataclass
+class SelectField(FieldBase):
+    """Single-choice dropdown."""
+    # Options can be a list or a callable that returns a list (for lazy evaluation)
+    options: Any = field(default_factory=list)  # [{value: "", label: ""}] or callable
+    filter_by_field: Optional[str] = None  # Field key whose value filters options via childOf property
+
+
+@dataclass
+class MultiSelectField(FieldBase):
+    """Multiple-choice selection."""
+    # Options can be a list or a callable that returns a list (for lazy evaluation)
+    options: Any = field(default_factory=list)  # [{value: "", label: ""}] or callable
+    default: List[str] = field(default_factory=list)
+    variant: str = "pills"  # "pills" (default) or "dropdown" for checkbox dropdown style
+
+
+@dataclass
+class OrderableListField(FieldBase):
+    # Options can be a list or a callable that returns a list (for lazy evaluation)
+    # Each option: {id, label, description?, disabledReason?, isLocked?, section?, isPinned?}
+    # - isLocked: toggle is disabled (can't enable/disable)
+    # - isPinned: can't be reordered (but toggle may still work if not also isLocked)
+    options: Any = field(default_factory=list)
+    # Default value: [{id, enabled}, ...] in priority order
+    default: List[Dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass
+class TableField(FieldBase):
+    """Editable table of structured rows."""
+
+    # Column definitions: [{key, label, type, placeholder?, options?, defaultValue?}, ...]
+    columns: Any = field(default_factory=list)  # list or callable
+
+    # Value format: list of objects
+    default: List[Dict[str, Any]] = field(default_factory=list)
+
+    add_label: str = "Add"
+    empty_message: str = ""
+
+
+@dataclass
+class ActionButton:
+    key: str                              # Action identifier
+    label: str                            # Button text
+    description: str = ""                 # Help text
+    style: str = "default"                # "default", "primary", "danger"
+    callback: Optional[Callable[..., Dict[str, Any]]] = None  # Returns {"success": bool, "message": str}
+    disabled: bool = False                # Whether button is disabled/greyed out
+    disabled_reason: str = ""             # Explanation shown when disabled
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    disabled_when: Optional[Dict[str, Any]] = None  # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
+
+    def get_field_type(self) -> str:
+        return "ActionButton"
+
+
+@dataclass
+class HeadingField:
+    """
+    Display-only heading with title and description.
+
+    Used to add section titles and descriptive text to settings pages.
+    Not an input field - purely for display.
+    """
+    key: str                              # Unique identifier
+    title: str                            # Heading title
+    description: str = ""                 # Description text (supports markdown-style links)
+    link_url: str = ""                    # Optional URL for a link
+    link_text: str = ""                   # Text for the link (defaults to URL if not provided)
+    show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None  # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
+    universal_only: bool = False          # Only show in Universal search mode (hide in Direct mode)
+
+    def get_field_type(self) -> str:
+        return "HeadingField"
+
+
+# Type alias for all field types
+SettingsField = Union[TextField, PasswordField, NumberField, CheckboxField, SelectField, MultiSelectField, OrderableListField, ActionButton, HeadingField]
+
+
+@dataclass
+class SettingsTab:
+    """A tab/section in the settings UI."""
+    name: str                             # Internal name (used in URLs)
+    display_name: str                     # Display name in UI
+    fields: List[SettingsField] = field(default_factory=list)
+    icon: Optional[str] = None            # Icon name for UI
+    order: int = 100                      # Sort order (lower = earlier)
+    group: Optional[str] = None           # Group name this tab belongs to
+
+
+@dataclass
+class SettingsGroup:
+    """A collapsible group of settings tabs in the UI."""
+    name: str                             # Internal name
+    display_name: str                     # Display name in UI
+    icon: Optional[str] = None            # Icon name for UI
+    order: int = 100                      # Sort order (lower = earlier)
+
+
+_SETTINGS_REGISTRY: Dict[str, SettingsTab] = {}
+_GROUPS_REGISTRY: Dict[str, SettingsGroup] = {}
+_ON_SAVE_HANDLERS: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {}
+_REGISTRY_LOCK = Lock()
+
+
+def register_group(
+    name: str,
+    display_name: str,
+    icon: Optional[str] = None,
+    order: int = 100
+) -> None:
+    with _REGISTRY_LOCK:
+        group = SettingsGroup(
+            name=name,
+            display_name=display_name,
+            icon=icon,
+            order=order,
+        )
+        _GROUPS_REGISTRY[name] = group
+        logger.debug(f"Registered settings group: {name}")
+
+
+def register_settings(
+    name: str,
+    display_name: str,
+    icon: Optional[str] = None,
+    order: int = 100,
+    group: Optional[str] = None
+):
+    def decorator(func: Callable[[], List[SettingsField]]):
+        with _REGISTRY_LOCK:
+            fields = func()
+            tab = SettingsTab(
+                name=name,
+                display_name=display_name,
+                fields=fields,
+                icon=icon,
+                order=order,
+                group=group,
+            )
+            _SETTINGS_REGISTRY[name] = tab
+            logger.debug(f"Registered settings tab: {name} ({len(fields)} fields)" +
+                        (f" in group {group}" if group else ""))
+        return func
+    return decorator
+
+
+def register_on_save(
+    tab_name: str,
+    handler: Callable[[Dict[str, Any]], Dict[str, Any]]
+) -> None:
+    with _REGISTRY_LOCK:
+        _ON_SAVE_HANDLERS[tab_name] = handler
+        logger.debug(f"Registered on_save handler for tab: {tab_name}")
+
+
+def get_on_save_handler(tab_name: str) -> Optional[Callable[[Dict[str, Any]], Dict[str, Any]]]:
+    """Get the on_save handler for a settings tab, if any."""
+    return _ON_SAVE_HANDLERS.get(tab_name)
+
+
+def get_settings_tab(name: str) -> Optional[SettingsTab]:
+    """Get a specific settings tab by name."""
+    return _SETTINGS_REGISTRY.get(name)
+
+
+def get_all_settings_tabs() -> List[SettingsTab]:
+    """Get all registered settings tabs, sorted by order."""
+    return sorted(_SETTINGS_REGISTRY.values(), key=lambda t: (t.order, t.name))
+
+
+def list_registered_settings() -> List[str]:
+    """List all registered settings tab names."""
+    return list(_SETTINGS_REGISTRY.keys())
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def _get_config_file_path(tab_name: str) -> Path:
+    """Get the config file path for a settings tab."""
+    config_dir = _get_config_dir()
+    # Core settings tabs share the main settings.json file
+    if tab_name in ("general", "search_mode"):
+        return config_dir / "settings.json"
+    return config_dir / "plugins" / f"{tab_name}.json"
+
+
+def _ensure_config_dir(tab_name: str) -> None:
+    """Ensure the config directory exists."""
+    config_path = _get_config_file_path(tab_name)
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def load_config_file(tab_name: str) -> Dict[str, Any]:
+    config_path = _get_config_file_path(tab_name)
+
+    if not config_path.exists():
+        return {}
+
+    try:
+        with open(config_path, 'r') as f:
+            return json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(f"Invalid JSON in config file {config_path}: {e}")
+        return {}
+
+
+def save_config_file(tab_name: str, values: Dict[str, Any]) -> bool:
+    try:
+        _ensure_config_dir(tab_name)
+        config_path = _get_config_file_path(tab_name)
+
+        # Load existing config and merge
+        existing = load_config_file(tab_name)
+        existing.update(values)
+
+        with open(config_path, 'w') as f:
+            json.dump(existing, f, indent=2)
+
+        logger.info(f"Saved settings to {config_path}")
+        return True
+    except Exception as e:
+        logger.error(f"Error saving config file for {tab_name}: {e}")
+        return False
+
+
+def initialize_default_configs() -> bool:
+    """Initialize config files with default values on first startup.
+
+    Creates config files for all settings tabs that don't have one yet,
+    populating them with field default values. This ensures config files
+    exist from first startup rather than only being created on explicit save.
+
+    Returns:
+        True if initialization succeeded or was skipped (already initialized),
+        False if there was an error accessing the config directory.
+    """
+    try:
+        config_dir = _get_config_dir()
+
+        # Check if config directory exists and is writable
+        if not config_dir.exists():
+            logger.warning(f"Config directory does not exist: {config_dir}")
+            return False
+
+        # Test writability
+        test_file = config_dir / ".write_test"
+        try:
+            test_file.touch()
+            test_file.unlink()
+        except (OSError, PermissionError) as e:
+            logger.warning(f"Config directory is not writable: {config_dir} - {e}")
+            return False
+
+        initialized_tabs = []
+
+        for tab in get_all_settings_tabs():
+            config_path = _get_config_file_path(tab.name)
+
+            # Skip if config file already exists
+            if config_path.exists():
+                continue
+
+            # Collect default values for all fields
+            defaults = {}
+            for field in tab.fields:
+                # Skip non-value fields
+                if isinstance(field, (ActionButton, HeadingField)):
+                    continue
+
+                # Only include fields that have a non-None default
+                if field.default is not None:
+                    defaults[field.key] = field.default
+
+            # Create config file with defaults if we have any
+            if defaults:
+                _ensure_config_dir(tab.name)
+                try:
+                    with open(config_path, 'w') as f:
+                        json.dump(defaults, f, indent=2)
+                    initialized_tabs.append(tab.name)
+                except Exception as e:
+                    logger.error(f"Failed to initialize config for {tab.name}: {e}")
+
+        if initialized_tabs:
+            logger.info(f"Initialized default configs for: {initialized_tabs}")
+
+        return True
+
+    except Exception as e:
+        logger.error(f"Error during config initialization: {e}")
+        return False
+
+
+def sync_env_to_config() -> None:
+    # Initialize default configs first (for fresh installs)
+    initialize_default_configs()
+
+    for tab in get_all_settings_tabs():
+        values_to_sync = {}
+
+        for field in tab.fields:
+            # Skip non-value fields
+            if isinstance(field, (ActionButton, HeadingField)):
+                continue
+
+            # Skip fields that don't support ENV vars
+            if not getattr(field, 'env_supported', True):
+                continue
+
+            # Check if ENV var is set
+            env_var_name = field.get_env_var_name()
+            env_value = os.environ.get(env_var_name)
+
+            if env_value is not None:
+                # Parse the ENV value to the appropriate type
+                parsed_value = _parse_env_value(env_value, field)
+                values_to_sync[field.key] = parsed_value
+
+        # Save synced values to config file (merge with existing)
+        if values_to_sync:
+            save_config_file(tab.name, values_to_sync)
+            logger.debug(f"Synced {len(values_to_sync)} ENV values to {tab.name} config: {list(values_to_sync.keys())}")
+
+    migrate_legacy_settings()
+
+
+def migrate_legacy_settings() -> None:
+    """Migrate legacy settings to new unified file destination format.
+
+    Maps old settings to new:
+    - PROCESSING_MODE + USE_BOOK_TITLE -> FILE_ORGANIZATION
+    - INGEST_DIR / LIBRARY_PATH -> DESTINATION
+    - LIBRARY_TEMPLATE -> TEMPLATE
+    - USE_CONTENT_TYPE_DIRECTORIES -> AA_CONTENT_TYPE_ROUTING
+    - INGEST_DIR_* -> AA_CONTENT_TYPE_DIR_*
+    - TORRENT_HARDLINK -> HARDLINK_TORRENTS / HARDLINK_TORRENTS_AUDIOBOOK
+    """
+    # Load existing downloads config
+    downloads_config = load_config_file("downloads")
+    source_config = load_config_file("download_sources")
+
+    # Skip migration if already using new settings
+    if "FILE_ORGANIZATION" in downloads_config or "DESTINATION" in downloads_config:
+        return
+
+    # Skip migration if no legacy settings exist (fresh install)
+    legacy_keys = {
+        "PROCESSING_MODE", "INGEST_DIR", "LIBRARY_PATH", "USE_BOOK_TITLE",
+        "LIBRARY_TEMPLATE", "PROCESSING_MODE_AUDIOBOOK", "INGEST_DIR_AUDIOBOOK",
+        "LIBRARY_PATH_AUDIOBOOK", "LIBRARY_TEMPLATE_AUDIOBOOK", "TORRENT_HARDLINK",
+        "USE_CONTENT_TYPE_DIRECTORIES",
+    }
+    if not any(key in downloads_config for key in legacy_keys):
+        return
+
+    migrated_downloads = {}
+    migrated_sources = {}
+
+    # === BOOKS MIGRATION ===
+    old_mode = downloads_config.get("PROCESSING_MODE", "ingest")
+    old_ingest_dir = downloads_config.get("INGEST_DIR", "/cwa-book-ingest")
+    old_library_path = downloads_config.get("LIBRARY_PATH", "")
+    old_use_book_title = downloads_config.get("USE_BOOK_TITLE", True)
+    old_library_template = downloads_config.get("LIBRARY_TEMPLATE", "{Author}/{Title}")
+
+    # Map PROCESSING_MODE + USE_BOOK_TITLE -> FILE_ORGANIZATION
+    if old_mode == "library":
+        migrated_downloads["FILE_ORGANIZATION"] = "organize"
+        migrated_downloads["DESTINATION"] = old_library_path or "/books"
+        migrated_downloads["TEMPLATE"] = old_library_template
+    else:
+        if old_use_book_title:
+            migrated_downloads["FILE_ORGANIZATION"] = "rename"
+            migrated_downloads["TEMPLATE"] = "{Author} - {Title} ({Year})"
+        else:
+            migrated_downloads["FILE_ORGANIZATION"] = "none"
+        migrated_downloads["DESTINATION"] = old_ingest_dir
+
+    # === AUDIOBOOKS MIGRATION ===
+    old_mode_ab = downloads_config.get("PROCESSING_MODE_AUDIOBOOK", "ingest")
+    old_ingest_dir_ab = downloads_config.get("INGEST_DIR_AUDIOBOOK", "")
+    old_library_path_ab = downloads_config.get("LIBRARY_PATH_AUDIOBOOK", "")
+    old_library_template_ab = downloads_config.get("LIBRARY_TEMPLATE_AUDIOBOOK", "{Author}/{Title}")
+
+    if old_mode_ab == "library":
+        migrated_downloads["FILE_ORGANIZATION_AUDIOBOOK"] = "organize"
+        migrated_downloads["DESTINATION_AUDIOBOOK"] = old_library_path_ab or ""
+        migrated_downloads["TEMPLATE_AUDIOBOOK"] = old_library_template_ab
+    else:
+        migrated_downloads["FILE_ORGANIZATION_AUDIOBOOK"] = "rename"
+        migrated_downloads["TEMPLATE_AUDIOBOOK"] = "{Author} - {Title}"
+        if old_ingest_dir_ab:
+            migrated_downloads["DESTINATION_AUDIOBOOK"] = old_ingest_dir_ab
+
+    # === HARDLINK MIGRATION ===
+    old_torrent_hardlink = downloads_config.get("TORRENT_HARDLINK")
+    if old_torrent_hardlink is not None:
+        # Books default to False (ingest folder use case)
+        # Audiobooks default to True (library folder use case)
+        # But if explicitly set, apply to both
+        migrated_downloads["HARDLINK_TORRENTS"] = old_torrent_hardlink
+        migrated_downloads["HARDLINK_TORRENTS_AUDIOBOOK"] = old_torrent_hardlink
+
+    # === CONTENT-TYPE ROUTING MIGRATION ===
+    old_use_content_type = downloads_config.get("USE_CONTENT_TYPE_DIRECTORIES", False)
+    if old_use_content_type:
+        migrated_sources["AA_CONTENT_TYPE_ROUTING"] = True
+
+        # Map old keys to new keys
+        content_type_mapping = {
+            "INGEST_DIR_BOOK_FICTION": "AA_CONTENT_TYPE_DIR_FICTION",
+            "INGEST_DIR_BOOK_NON_FICTION": "AA_CONTENT_TYPE_DIR_NON_FICTION",
+            "INGEST_DIR_BOOK_UNKNOWN": "AA_CONTENT_TYPE_DIR_UNKNOWN",
+            "INGEST_DIR_MAGAZINE": "AA_CONTENT_TYPE_DIR_MAGAZINE",
+            "INGEST_DIR_COMIC_BOOK": "AA_CONTENT_TYPE_DIR_COMIC",
+            "INGEST_DIR_STANDARDS_DOCUMENT": "AA_CONTENT_TYPE_DIR_STANDARDS",
+            "INGEST_DIR_MUSICAL_SCORE": "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+            "INGEST_DIR_OTHER": "AA_CONTENT_TYPE_DIR_OTHER",
+        }
+
+        for old_key, new_key in content_type_mapping.items():
+            old_value = downloads_config.get(old_key, "")
+            if old_value:
+                migrated_sources[new_key] = old_value
+
+    # Save migrated settings
+    if migrated_downloads:
+        save_config_file("downloads", migrated_downloads)
+        logger.info(f"Migrated download settings: {list(migrated_downloads.keys())}")
+
+    if migrated_sources:
+        save_config_file("download_sources", migrated_sources)
+        logger.info(f"Migrated content-type routing settings: {list(migrated_sources.keys())}")
+
+
+def get_setting_value(field: SettingsField, tab_name: str) -> Any:
+    if isinstance(field, (ActionButton, HeadingField)):
+        return None  # Actions and headings don't have values
+
+    # 1. Check environment variable (if supported for this field)
+    if field.env_supported:
+        env_var_name = field.get_env_var_name()
+        env_value = os.environ.get(env_var_name)
+        if env_value is not None:
+            return _parse_env_value(env_value, field)
+
+    # 2. Check config file
+    config = load_config_file(tab_name)
+    if field.key in config:
+        return config[field.key]
+
+    # 3. Return default
+    return field.default
+
+
+def _parse_env_value(value: str, field: SettingsField) -> Any:
+    """Parse an environment variable value to the appropriate type."""
+    if isinstance(field, CheckboxField):
+        return value.lower() in ('true', '1', 'yes', 'on')
+    elif isinstance(field, NumberField):
+        try:
+            if '.' in value:
+                return float(value)
+            return int(value)
+        except ValueError:
+            return field.default
+    elif isinstance(field, MultiSelectField):
+        return [v.strip() for v in value.split(',') if v.strip()]
+    elif isinstance(field, OrderableListField):
+        # Parse JSON array: [{"id": "...", "enabled": true}, ...]
+        try:
+            return json.loads(value)
+        except json.JSONDecodeError:
+            logger.warning(f"Invalid JSON for {field.key}, using default")
+            return field.default
+    elif isinstance(field, TableField):
+        # Parse JSON array: [{"col": "value"}, ...]
+        try:
+            parsed = json.loads(value)
+            return parsed if isinstance(parsed, list) else field.default
+        except json.JSONDecodeError:
+            logger.warning(f"Invalid JSON for {field.key}, using default")
+            return field.default
+    else:
+        return value
+
+
+def is_value_from_env(field: SettingsField) -> bool:
+    """Check if a field's value comes from an environment variable."""
+    if isinstance(field, (ActionButton, HeadingField)):
+        return False
+    # UI-only settings never come from ENV (env_supported=False)
+    if not getattr(field, 'env_supported', True):
+        return False
+    return field.get_env_var_name() in os.environ
+
+
+def serialize_field(field: SettingsField, tab_name: str, include_value: bool = True) -> Dict[str, Any]:
+    """
+    Serialize a field for API response.
+
+    Args:
+        field: The settings field.
+        tab_name: The settings tab name.
+        include_value: Whether to include the current value.
+
+    Returns:
+        Dict representation of the field.
+    """
+    # HeadingField has a different structure - handle separately
+    if isinstance(field, HeadingField):
+        result: Dict[str, Any] = {
+            "key": field.key,
+            "type": field.get_field_type(),
+            "title": field.title,
+            "description": field.description,
+        }
+        if field.link_url:
+            result["linkUrl"] = field.link_url
+            result["linkText"] = field.link_text or field.link_url
+        if field.show_when:
+            result["showWhen"] = field.show_when
+        if field.universal_only:
+            result["universalOnly"] = True
+        return result
+
+    result: Dict[str, Any] = {
+        "key": field.key,
+        "label": field.label,
+        "type": field.get_field_type(),
+        "description": getattr(field, 'description', ''),
+        "required": getattr(field, 'required', False),
+        "disabled": getattr(field, 'disabled', False),
+        "disabledReason": getattr(field, 'disabled_reason', ''),
+        "requiresRestart": getattr(field, 'requires_restart', False),
+    }
+
+    # Add optional properties if set
+    if getattr(field, 'show_when', None):
+        result["showWhen"] = field.show_when
+    if getattr(field, 'disabled_when', None):
+        result["disabledWhen"] = field.disabled_when
+    if getattr(field, 'universal_only', False):
+        result["universalOnly"] = True
+
+    # Add type-specific properties
+    if isinstance(field, TextField):
+        result["placeholder"] = field.placeholder
+        if field.max_length:
+            result["maxLength"] = field.max_length
+    elif isinstance(field, PasswordField):
+        result["placeholder"] = field.placeholder
+    elif isinstance(field, NumberField):
+        result["min"] = field.min_value
+        result["max"] = field.max_value
+        result["step"] = field.step
+    elif isinstance(field, SelectField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+        if field.default is not None:
+            result["default"] = field.default
+        if field.filter_by_field:
+            result["filterByField"] = field.filter_by_field
+    elif isinstance(field, MultiSelectField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+        result["variant"] = field.variant
+    elif isinstance(field, OrderableListField):
+        # Support callable options for lazy evaluation (avoids circular imports)
+        options = field.options() if callable(field.options) else field.options
+        result["options"] = options
+    elif isinstance(field, TableField):
+        columns = field.columns() if callable(field.columns) else field.columns
+        result["columns"] = columns
+        result["addLabel"] = field.add_label
+        result["emptyMessage"] = field.empty_message
+    elif isinstance(field, ActionButton):
+        result["style"] = field.style
+        result["description"] = field.description
+
+    if include_value and not isinstance(field, (ActionButton, HeadingField)):
+        value = get_setting_value(field, tab_name)
+
+        # Ensure select values are serialized as strings so the frontend can
+        # reliably match against string option values.
+        if isinstance(field, SelectField) and value is not None:
+            value = str(value)
+        elif isinstance(field, MultiSelectField):
+            if value is None:
+                value = []
+            elif isinstance(value, list):
+                value = [str(v) for v in value]
+            elif isinstance(value, str):
+                # Support legacy/manual configs where MultiSelect values were saved
+                # as comma-separated strings.
+                value = [v.strip() for v in value.split(",") if v.strip()]
+            else:
+                value = []
+        elif isinstance(field, TableField):
+            if value is None:
+                value = []
+            elif not isinstance(value, list):
+                value = []
+
+        result["value"] = value if value is not None else ""
+        result["fromEnv"] = is_value_from_env(field)
+
+    return result
+
+
+def serialize_tab(tab: SettingsTab, include_values: bool = True) -> Dict[str, Any]:
+    """Serialize a settings tab for API response."""
+    return {
+        "name": tab.name,
+        "displayName": tab.display_name,
+        "icon": tab.icon,
+        "order": tab.order,
+        "group": tab.group,
+        "fields": [serialize_field(f, tab.name, include_values) for f in tab.fields],
+    }
+
+
+def serialize_group(group: SettingsGroup) -> Dict[str, Any]:
+    """Serialize a settings group for API response."""
+    return {
+        "name": group.name,
+        "displayName": group.display_name,
+        "icon": group.icon,
+        "order": group.order,
+    }
+
+
+def get_all_groups() -> List[SettingsGroup]:
+    """Get all registered settings groups, sorted by order."""
+    return sorted(_GROUPS_REGISTRY.values(), key=lambda g: (g.order, g.name))
+
+
+def serialize_all_settings(include_values: bool = True) -> Dict[str, Any]:
+    """Serialize all settings for API response."""
+    tabs = get_all_settings_tabs()
+    groups = get_all_groups()
+    return {
+        "tabs": [serialize_tab(t, include_values) for t in tabs],
+        "groups": [serialize_group(g) for g in groups],
+    }
+
+
+def execute_action(tab_name: str, action_key: str, current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """
+    Execute an action button's callback.
+
+    Args:
+        tab_name: The settings tab name.
+        action_key: The action key to execute.
+        current_values: Optional dict of current form values (unsaved).
+                       Passed to callbacks that accept it.
+
+    Returns:
+        Dict with "success" (bool) and "message" (str).
+    """
+    import inspect
+
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        return {"success": False, "message": f"Unknown settings tab: {tab_name}"}
+
+    for field in tab.fields:
+        if isinstance(field, ActionButton) and field.key == action_key:
+            if field.callback:
+                try:
+                    # Check if callback accepts current_values parameter
+                    sig = inspect.signature(field.callback)
+                    if "current_values" in sig.parameters:
+                        return field.callback(current_values=current_values or {})
+                    else:
+                        return field.callback()
+                except Exception as e:
+                    logger.error(f"Action {action_key} failed: {e}")
+                    return {"success": False, "message": str(e)}
+            else:
+                return {"success": False, "message": "Action has no callback defined"}
+
+    return {"success": False, "message": f"Unknown action: {action_key}"}
+
+
+def _sync_metadata_provider_selection() -> None:
+    """
+    Sync the METADATA_PROVIDER setting based on enabled providers.
+
+    Called after saving metadata provider settings to auto-select
+    the first enabled provider if the current selection is invalid.
+    """
+    try:
+        from shelfmark.metadata_providers import sync_metadata_provider_selection
+        sync_metadata_provider_selection()
+    except ImportError:
+        pass  # Metadata providers module not available
+
+
+def _apply_dns_settings(config) -> None:
+    """
+    Apply DNS settings changes to the network module.
+
+    This ensures DNS changes take effect immediately without requiring
+    a container restart.
+    """
+    try:
+        from shelfmark.download import network
+
+        provider = config.get("CUSTOM_DNS", "auto")
+        use_doh = config.get("USE_DOH", False)
+        manual_servers = None
+
+        if provider == "manual":
+            manual_dns = config.get("CUSTOM_DNS_MANUAL", "")
+            if manual_dns:
+                # Parse comma-separated server list
+                manual_servers = [s.strip() for s in manual_dns.split(",") if s.strip()]
+
+        network.set_dns_provider(provider, manual_servers, use_doh=use_doh)
+    except ImportError:
+        pass  # Network module not available
+    except Exception as e:
+        logger.warning(f"Failed to apply DNS settings: {e}")
+
+
+def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]:
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        return {"success": False, "message": f"Unknown settings tab: {tab_name}", "updated": [], "requiresRestart": False}
+
+    # Build a map of field keys to fields (exclude non-value fields)
+    field_map = {f.key: f for f in tab.fields if not isinstance(f, (ActionButton, HeadingField))}
+
+    # Filter out values that are set via env vars or unknown
+    values_to_save = {}
+    skipped_env = []
+    skipped_unknown = []
+    restart_required_keys = []
+
+    for key, value in values.items():
+        if key not in field_map:
+            skipped_unknown.append(key)
+            continue
+
+        field = field_map[key]
+        if is_value_from_env(field):
+            skipped_env.append(key)
+            continue
+
+        # Handle password fields - only update if a new value is provided
+        if isinstance(field, PasswordField) and not value:
+            continue
+
+        values_to_save[key] = value
+
+        # Track if this field requires restart
+        if getattr(field, 'requires_restart', False):
+            restart_required_keys.append(key)
+
+    if not values_to_save:
+        message = "No settings to update"
+        if skipped_env:
+            message += f". Skipped (set via env): {', '.join(skipped_env)}"
+        return {"success": True, "message": message, "updated": [], "requiresRestart": False}
+
+    # Call on_save handler if registered (for custom validation/transformation)
+    on_save_handler = get_on_save_handler(tab_name)
+    if on_save_handler:
+        try:
+            result = on_save_handler(values_to_save.copy())
+            if result.get("error"):
+                return {
+                    "success": False,
+                    "message": result.get("message", "Validation failed"),
+                    "updated": [],
+                    "requiresRestart": False
+                }
+            # Use the transformed values
+            values_to_save = result.get("values", values_to_save)
+        except Exception as e:
+            logger.error(f"on_save handler for {tab_name} failed: {e}")
+            return {
+                "success": False,
+                "message": f"Save handler error: {str(e)}",
+                "updated": [],
+                "requiresRestart": False
+            }
+
+    # Save to config file
+    if save_config_file(tab_name, values_to_save):
+        # Refresh the config singleton so live settings take effect immediately
+        config_obj = None
+        try:
+            from shelfmark.core.config import config as config_obj
+
+            config_obj.refresh()
+        except ImportError:
+            config_obj = None  # Config module not yet available during initial setup
+
+        # Apply DNS settings changes live (network tab)
+        dns_keys = {"CUSTOM_DNS", "CUSTOM_DNS_MANUAL", "USE_DOH"}
+        if (
+            config_obj is not None
+            and tab_name == "network"
+            and dns_keys.intersection(values_to_save.keys())
+        ):
+            _apply_dns_settings(config_obj)
+
+        # Sync metadata provider selection when a provider's enabled state changes
+        tab = get_settings_tab(tab_name)
+        if tab and tab.group == "metadata_providers":
+            _sync_metadata_provider_selection()
+
+        message = f"Updated {len(values_to_save)} setting(s)"
+        if skipped_env:
+            message += f". Skipped (set via env): {', '.join(skipped_env)}"
+
+        requires_restart = len(restart_required_keys) > 0
+        return {
+            "success": True,
+            "message": message,
+            "updated": list(values_to_save.keys()),
+            "requiresRestart": requires_restart,
+            "restartRequiredFor": restart_required_keys,
+        }
+    else:
+        return {"success": False, "message": "Failed to save settings", "updated": [], "requiresRestart": False}

shelfmark/core/utils.py

@@ -0,0 +1,196 @@
+"""Shared utility functions for the Shelfmark."""
+
+import base64
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+
+def normalize_http_url(
+    url: Optional[str],
+    *,
+    default_scheme: str = "http",
+    strip_trailing_slash: bool = True,
+    allow_special: tuple[str, ...] = (),
+) -> str:
+    """Normalize a configured HTTP URL for requests and links."""
+    if not isinstance(url, str):
+        return ""
+
+    normalized = url.strip()
+    if not normalized:
+        return ""
+
+    if (normalized.startswith("\"") and normalized.endswith("\"")) or (
+        normalized.startswith("'") and normalized.endswith("'")
+    ):
+        normalized = normalized[1:-1].strip()
+        if not normalized:
+            return ""
+
+    if allow_special:
+        special_map = {
+            value.lower(): value
+            for value in allow_special
+            if isinstance(value, str)
+        }
+        special_match = special_map.get(normalized.lower())
+        if special_match is not None:
+            return special_match
+
+    if normalized.startswith(("/", "./", "../")):
+        return normalized
+
+    if "://" not in normalized:
+        scheme = default_scheme.strip().rstrip(":/")
+        if scheme:
+            normalized = f"{scheme}://{normalized}"
+
+    if strip_trailing_slash:
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def normalize_base_path(value: Optional[str]) -> str:
+    """Normalize a URL base path for reverse proxy subpath deployments."""
+    if not isinstance(value, str):
+        return ""
+
+    path = value.strip()
+    if not path:
+        return ""
+
+    if "://" in path:
+        parsed = urlparse(path)
+        path = parsed.path or ""
+
+    if not path or path == "/":
+        return ""
+
+    if not path.startswith("/"):
+        path = "/" + path
+
+    return path.rstrip("/")
+
+
+def is_audiobook(content_type: Optional[str]) -> bool:
+    """Check if content type indicates an audiobook."""
+    return bool(content_type and "audiobook" in content_type.lower())
+
+
+CONTENT_TYPES = [
+    "book (fiction)",
+    "book (non-fiction)",
+    "book (unknown)",
+    "magazine",
+    "comic book",
+    "audiobook",
+    "standards document",
+    "musical score",
+    "other",
+]
+
+# Maps AA content types to their config keys for content-type routing
+# Used when AA_CONTENT_TYPE_ROUTING is enabled
+_AA_CONTENT_TYPE_TO_CONFIG_KEY = {
+    "book (fiction)": "AA_CONTENT_TYPE_DIR_FICTION",
+    "book (non-fiction)": "AA_CONTENT_TYPE_DIR_NON_FICTION",
+    "book (unknown)": "AA_CONTENT_TYPE_DIR_UNKNOWN",
+    "magazine": "AA_CONTENT_TYPE_DIR_MAGAZINE",
+    "comic book": "AA_CONTENT_TYPE_DIR_COMIC",
+    "audiobook": "AA_CONTENT_TYPE_DIR_AUDIOBOOK",
+    "standards document": "AA_CONTENT_TYPE_DIR_STANDARDS",
+    "musical score": "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+    "other": "AA_CONTENT_TYPE_DIR_OTHER",
+}
+
+# Legacy mapping - kept for backwards compatibility during migration
+_LEGACY_CONTENT_TYPE_TO_CONFIG_KEY = {
+    "book (fiction)": "INGEST_DIR_BOOK_FICTION",
+    "book (non-fiction)": "INGEST_DIR_BOOK_NON_FICTION",
+    "book (unknown)": "INGEST_DIR_BOOK_UNKNOWN",
+    "magazine": "INGEST_DIR_MAGAZINE",
+    "comic book": "INGEST_DIR_COMIC_BOOK",
+    "audiobook": "INGEST_DIR_AUDIOBOOK",
+    "standards document": "INGEST_DIR_STANDARDS_DOCUMENT",
+    "musical score": "INGEST_DIR_MUSICAL_SCORE",
+    "other": "INGEST_DIR_OTHER",
+}
+
+
+def get_destination(is_audiobook: bool = False) -> Path:
+    """Get base destination directory. Audiobooks fall back to main destination."""
+    from shelfmark.core.config import config
+
+    if is_audiobook:
+        # Audiobook destination with fallback to main destination
+        audiobook_dest = config.get("DESTINATION_AUDIOBOOK", "")
+        if audiobook_dest:
+            return Path(audiobook_dest)
+
+    # Main destination (also fallback for audiobooks)
+    # Check new setting first, then legacy INGEST_DIR
+    destination = config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books")
+    return Path(destination)
+
+
+def get_aa_content_type_dir(content_type: Optional[str] = None) -> Optional[Path]:
+    """Get override directory for AA content-type routing if configured."""
+    from shelfmark.core.config import config
+
+    # Check if content-type routing is enabled (new or legacy setting)
+    if not config.get("AA_CONTENT_TYPE_ROUTING", False) and not config.get("USE_CONTENT_TYPE_DIRECTORIES", False):
+        return None
+
+    if not content_type:
+        return None
+
+    content_type_lower = content_type.lower().strip()
+
+    # Try new AA-specific config keys first, then legacy keys
+    for mapping in (_AA_CONTENT_TYPE_TO_CONFIG_KEY, _LEGACY_CONTENT_TYPE_TO_CONFIG_KEY):
+        config_key = mapping.get(content_type_lower)
+        if config_key:
+            custom_dir = config.get(config_key, "")
+            if custom_dir:
+                return Path(custom_dir)
+
+    return None
+
+
+def get_ingest_dir(content_type: Optional[str] = None) -> Path:
+    """DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead."""
+    from shelfmark.core.config import config
+
+    # Check new DESTINATION setting first, then legacy INGEST_DIR
+    default_ingest_dir = Path(config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books"))
+
+    if not content_type:
+        return default_ingest_dir
+
+    # Check for content-type override
+    override_dir = get_aa_content_type_dir(content_type)
+    if override_dir:
+        return override_dir
+
+    return default_ingest_dir
+
+
+def transform_cover_url(cover_url: Optional[str], cache_id: str) -> Optional[str]:
+    """Transform external cover URL to local proxy URL when caching is enabled."""
+    if not cover_url:
+        return cover_url
+
+    # Skip if already a local URL (starts with /)
+    if cover_url.startswith('/'):
+        return cover_url
+
+    # Check if cover caching is enabled
+    from shelfmark.config.env import is_covers_cache_enabled
+    if not is_covers_cache_enabled():
+        return cover_url
+
+    # Encode the original URL and create a proxy URL
+    encoded_url = base64.urlsafe_b64encode(cover_url.encode()).decode()
+    return f"/api/covers/{cache_id}?url={encoded_url}"

shelfmark/download/__init__.py

@@ -0,0 +1 @@
+"""Download module - HTTP downloads, network, and orchestration."""

shelfmark/download/archive.py

@@ -0,0 +1,239 @@
+"""Archive extraction utilities for downloaded book archives."""
+
+import os
+import shutil
+import zipfile
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.postprocess.policy import (
+    get_supported_audiobook_formats,
+    get_supported_formats,
+)
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.fs import atomic_write
+
+logger = setup_logger(__name__)
+
+
+# Check for rarfile availability at module load
+try:
+    import rarfile
+
+    RAR_AVAILABLE = True
+except ImportError:
+    RAR_AVAILABLE = False
+    logger.warning("rarfile not installed - RAR extraction disabled")
+
+
+class ArchiveExtractionError(Exception):
+    """Raised when archive extraction fails."""
+
+    pass
+
+
+class PasswordProtectedError(ArchiveExtractionError):
+    """Raised when archive requires a password."""
+
+    pass
+
+
+class CorruptedArchiveError(ArchiveExtractionError):
+    """Raised when archive is corrupted."""
+
+    pass
+
+
+def is_archive(file_path: Path) -> bool:
+    """Check if file is a supported archive format."""
+    suffix = file_path.suffix.lower().lstrip(".")
+    return suffix in ("zip", "rar")
+
+
+def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> bool:
+    """Check if file matches user's supported formats setting based on content type."""
+    ext = file_path.suffix.lower().lstrip(".")
+    if check_audiobook(content_type):
+        supported_formats = get_supported_audiobook_formats()
+    else:
+        supported_formats = get_supported_formats()
+    return ext in supported_formats
+
+
+# All known ebook extensions (superset of what user might enable)
+ALL_EBOOK_EXTENSIONS = {'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr', '.doc', '.docx', '.rtf', '.txt'}
+
+# All known audio extensions (superset of what user might enable for audiobooks)
+ALL_AUDIO_EXTENSIONS = {'.m4b', '.mp3', '.m4a', '.aac', '.flac', '.ogg', '.wma', '.wav', '.opus'}
+
+
+def _filter_files(
+    extracted_files: List[Path],
+    content_type: Optional[str] = None,
+) -> Tuple[List[Path], List[Path], List[Path]]:
+    """Filter files by content type. Returns (matched, rejected_format, other)."""
+    is_audiobook = check_audiobook(content_type)
+    known_extensions = ALL_AUDIO_EXTENSIONS if is_audiobook else ALL_EBOOK_EXTENSIONS
+
+    matched_files = []
+    rejected_format_files = []
+    other_files = []
+
+    for file_path in extracted_files:
+        if _is_supported_file(file_path, content_type):
+            matched_files.append(file_path)
+        elif file_path.suffix.lower() in known_extensions:
+            rejected_format_files.append(file_path)
+        else:
+            other_files.append(file_path)
+
+    return matched_files, rejected_format_files, other_files
+
+
+def extract_archive(
+    archive_path: Path,
+    output_dir: Path,
+    content_type: Optional[str] = None,
+) -> Tuple[List[Path], List[str], List[Path]]:
+    """Extract archive and filter by content type. Returns (matched, warnings, rejected)."""
+    suffix = archive_path.suffix.lower().lstrip(".")
+
+    if suffix == "zip":
+        extracted_files, warnings = _extract_zip(archive_path, output_dir)
+    elif suffix == "rar":
+        extracted_files, warnings = _extract_rar(archive_path, output_dir)
+    else:
+        raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
+
+    is_audiobook = check_audiobook(content_type)
+    file_type_label = "audiobook" if is_audiobook else "book"
+
+    # Filter files based on content type
+    matched_files, rejected_files, other_files = _filter_files(extracted_files, content_type)
+
+    # Delete rejected files (valid formats but not enabled by user)
+    for rejected_file in rejected_files:
+        try:
+            rejected_file.unlink()
+            logger.debug(f"Deleted rejected {file_type_label} file: {rejected_file.name}")
+        except OSError as e:
+            logger.warning(f"Failed to delete rejected {file_type_label} file {rejected_file}: {e}")
+
+    if rejected_files:
+        rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+        warnings.append(f"Skipped {len(rejected_files)} {file_type_label}(s) with unsupported format: {', '.join(rejected_exts)}")
+
+    # Delete other files (images, html, etc)
+    for other_file in other_files:
+        try:
+            other_file.unlink()
+            logger.debug(f"Deleted non-{file_type_label} file: {other_file.name}")
+        except OSError as e:
+            logger.warning(f"Failed to delete non-{file_type_label} file {other_file}: {e}")
+
+    if other_files:
+        warnings.append(f"Skipped {len(other_files)} non-{file_type_label} file(s)")
+
+    return matched_files, warnings, rejected_files
+
+
+def extract_archive_raw(
+    archive_path: Path,
+    output_dir: Path,
+) -> Tuple[List[Path], List[str]]:
+    """Extract archive without filtering (returns all extracted files)."""
+    suffix = archive_path.suffix.lower().lstrip(".")
+
+    if suffix == "zip":
+        return _extract_zip(archive_path, output_dir)
+    if suffix == "rar":
+        return _extract_rar(archive_path, output_dir)
+
+    raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
+
+
+def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
+    """Extract files from ZipFile or RarFile to output_dir with security checks."""
+    extracted_files = []
+
+    for info in archive.infolist():
+        if info.is_dir():
+            continue
+
+        # Use only filename, strip directory path (security: prevent path traversal)
+        filename = Path(info.filename).name
+        if not filename:
+            continue
+
+        # Security: reject filenames with null bytes or path separators
+        # Check both / and \ since archives may be created on different OSes
+        if "\x00" in filename or "/" in filename or "\\" in filename:
+            logger.warning(f"Skipping suspicious filename in archive: {info.filename!r}")
+            continue
+
+        # Extract to output_dir with flat structure
+        target_path = output_dir / filename
+
+        # Security: verify resolved path stays within output directory (defense-in-depth)
+        try:
+            target_path.resolve().relative_to(output_dir.resolve())
+        except ValueError:
+            logger.warning(f"Path traversal attempt blocked: {info.filename!r}")
+            continue
+
+        with archive.open(info) as src:
+            data = src.read()
+        final_path = atomic_write(target_path, data)
+        extracted_files.append(final_path)
+        logger.debug(f"Extracted: {filename}")
+
+    return extracted_files
+
+
+def _extract_zip(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
+    """Extract files from a ZIP archive."""
+    try:
+        with zipfile.ZipFile(archive_path, "r") as zf:
+            # Check for password protection
+            for info in zf.infolist():
+                if info.flag_bits & 0x1:  # Encrypted flag
+                    raise PasswordProtectedError("ZIP archive is password protected")
+
+            # Test archive integrity
+            bad_file = zf.testzip()
+            if bad_file:
+                raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
+
+            return _extract_files_from_archive(zf, output_dir), []
+
+    except zipfile.BadZipFile as e:
+        raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
+    except PermissionError as e:
+        raise ArchiveExtractionError(f"Permission denied: {e}")
+
+
+def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
+    """Extract files from a RAR archive."""
+    if not RAR_AVAILABLE:
+        raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
+
+    try:
+        with rarfile.RarFile(archive_path, "r") as rf:
+            # Check for password protection
+            if rf.needs_password():
+                raise PasswordProtectedError("RAR archive is password protected")
+
+            # Test archive integrity
+            rf.testrar()
+
+            return _extract_files_from_archive(rf, output_dir), []
+
+    except rarfile.BadRarFile as e:
+        raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
+    except rarfile.RarCannotExec:
+        raise ArchiveExtractionError("unrar binary not found - install unrar package")
+    except PermissionError as e:
+        raise ArchiveExtractionError(f"Permission denied: {e}")
+
+

shelfmark/download/fs.py

@@ -0,0 +1,409 @@
+"""Atomic filesystem operations for concurrent-safe file handling.
+
+These utilities handle file collisions atomically, avoiding TOCTOU race conditions
+when multiple workers may try to write to the same path simultaneously.
+"""
+
+import errno
+import os
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.permissions_debug import log_transfer_permission_context
+
+logger = setup_logger(__name__)
+
+
+
+_VERIFY_IO_WAIT_SECONDS = 3.0
+
+
+def _verify_transfer_size(
+    dest: Path,
+    expected_size: int,
+    action: str,
+) -> None:
+    """Verify file transfer completed successfully.
+
+    Some filesystems (especially remote NAS/CIFS/NFS) can report stale sizes briefly
+    after large writes. Do a second stat after a short delay before declaring failure.
+    """
+    actual_size = dest.stat().st_size
+    if actual_size == expected_size:
+        return
+
+    logger.debug(
+        f"File {action} size mismatch, waiting for filesystem sync: {dest} "
+        f"({actual_size} != {expected_size})"
+    )
+    time.sleep(_VERIFY_IO_WAIT_SECONDS)
+
+    actual_size = dest.stat().st_size
+    if actual_size != expected_size:
+        raise IOError(
+            f"File {action} incomplete, data loss may have occurred. "
+            f"'{dest}' was {actual_size} bytes instead of expected {expected_size}."
+        )
+
+
+def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
+    """Write data to a file with atomic collision detection.
+
+    If the destination already exists, retries with counter suffix (_1, _2, etc.)
+    until a unique path is found.
+
+    Args:
+        dest_path: Desired destination path
+        data: Bytes to write
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually written (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            # O_CREAT | O_EXCL fails atomically if file exists
+            fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+            try:
+                os.write(fd, data)
+            finally:
+                os.close(fd)
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            continue
+
+    raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}")
+
+
+def _is_permission_error(e: Exception) -> bool:
+    """Check if exception is a permission error (including NFS/SMB issues)."""
+    return isinstance(e, PermissionError) or (isinstance(e, OSError) and e.errno == errno.EPERM)
+
+
+def _system_op(op: str, source: Path, dest: Path) -> None:
+    """Execute system command (mv or cp) as final fallback."""
+    logger.warning("Attempting system %s as final fallback: %s -> %s", op, source, dest)
+    subprocess.run(
+        [op, "-f", str(source), str(dest)],
+        check=True,
+        capture_output=True,
+        text=True
+    )
+
+
+def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
+    """Handle NFS/SMB permission errors by falling back to copyfile -> system op."""
+    expected_size = source.stat().st_size
+
+    try:
+        # Fallback 1: copy content only
+        shutil.copyfile(str(source), str(dest))
+        _verify_transfer_size(dest, expected_size, "copy")
+
+        if is_move:
+            source.unlink()
+        return
+
+    except Exception as copy_error:
+        # Clean up failed copy attempt if it exists
+        dest.unlink(missing_ok=True)
+
+        if _is_permission_error(copy_error):
+            log_transfer_permission_context("nfs_fallback_copyfile", source=source, dest=dest, error=copy_error)
+        logger.error("Fallback copyfile failed (%s -> %s): %s", source, dest, copy_error)
+
+        # Fallback 2: system command
+        op = "mv" if is_move else "cp"
+        try:
+            _system_op(op, source, dest)
+            # Best-effort verify after external command.
+            if dest.exists():
+                _verify_transfer_size(dest, expected_size, op)
+            if is_move:
+                source.unlink(missing_ok=True)
+        except subprocess.CalledProcessError as sys_error:
+            log_transfer_permission_context("nfs_fallback_system", source=source, dest=dest, error=sys_error)
+            logger.error("System %s failed (%s -> %s): %s", op, source, dest, sys_error.stderr)
+            dest.unlink(missing_ok=True)
+            raise
+
+
+def _claim_destination(path: Path) -> bool:
+    """Atomically claim a destination path by creating a placeholder file.
+
+    Returns True if the placeholder was created. Caller must replace or unlink it.
+    """
+    try:
+        fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+    except FileExistsError:
+        return False
+    else:
+        os.close(fd)
+        return True
+
+
+def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Move a file with collision detection.
+
+    Uses os.rename() for same-filesystem moves (atomic, triggers inotify events),
+    falls back to exclusive create + shutil.move for cross-filesystem moves.
+
+    Note: We use os.rename() instead of hardlink+unlink because os.rename()
+    triggers proper inotify IN_MOVED_TO events that file watchers (like Calibre's
+    auto-add) rely on to detect new files.
+
+    Args:
+        source_path: Source file to move
+        dest_path: Desired destination path
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually moved (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+
+        # Check for existing file (os.rename would overwrite on Unix)
+        claimed = False
+        if try_path.exists():
+            # Some filesystems can report false positives for exists() with
+            # special characters. Probe with O_EXCL to confirm.
+            claimed = _claim_destination(try_path)
+            if not claimed:
+                continue
+
+        try:
+            # os.rename is atomic on same filesystem and triggers inotify events
+            if claimed:
+                os.replace(str(source_path), str(try_path))
+            else:
+                os.rename(str(source_path), str(try_path))
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            # Race condition: file created between exists() check and rename()
+            if claimed:
+                try_path.unlink(missing_ok=True)
+            continue
+        except OSError as e:
+            # Cross-filesystem - fall back to exclusive create + verified copy + delete.
+            if e.errno != errno.EXDEV:
+                if claimed:
+                    try_path.unlink(missing_ok=True)
+                raise
+
+            expected_size = source_path.stat().st_size
+
+            try:
+                if not claimed:
+                    # Claim destination path atomically.
+                    fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+                    os.close(fd)
+
+                # Copy to a temp file first, then replace to avoid partial files.
+                temp_path = try_path.parent / f".{try_path.name}.tmp"
+                try:
+                    try:
+                        shutil.copy2(str(source_path), str(temp_path))
+                    except (PermissionError, OSError) as copy_error:
+                        if _is_permission_error(copy_error):
+                            logger.debug(
+                                "Permission error during move-copy, falling back to copyfile (%s -> %s): %s",
+                                source_path,
+                                temp_path,
+                                copy_error,
+                            )
+                            _perform_nfs_fallback(source_path, temp_path, is_move=False)
+                        else:
+                            raise
+
+                    temp_path.replace(try_path)
+                    _verify_transfer_size(try_path, expected_size, "move")
+                    source_path.unlink()
+
+                    if attempt > 0:
+                        logger.info(f"File collision resolved: {try_path.name}")
+                    return try_path
+
+                except Exception:
+                    try_path.unlink(missing_ok=True)
+                    temp_path.unlink(missing_ok=True)
+                    raise
+
+            except FileExistsError:
+                continue
+            except (PermissionError, OSError) as e:
+                if _is_permission_error(e):
+                    log_transfer_permission_context(
+                        "atomic_move",
+                        source=source_path,
+                        dest=try_path,
+                        error=e,
+                    )
+                    logger.debug(
+                        "Permission error during move, falling back to copyfile (%s -> %s): %s",
+                        source_path,
+                        try_path,
+                        e,
+                    )
+                    try:
+                        _perform_nfs_fallback(source_path, try_path, is_move=True)
+                        if attempt > 0:
+                            logger.info(f"File collision resolved (fallback): {try_path.name}")
+                        return try_path
+                    except Exception as fallback_error:
+                        logger.error(
+                            "NFS fallback also failed (%s -> %s): %s",
+                            source_path,
+                            try_path,
+                            fallback_error,
+                        )
+                        raise e from fallback_error
+                raise
+
+    raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
+
+
+def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Create a hardlink with atomic collision detection.
+
+    Args:
+        source_path: Source file to link from
+        dest_path: Desired destination path for the link
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where link was actually created (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            os.link(str(source_path), str(try_path))
+            if attempt > 0:
+                logger.info(f"File collision resolved: {try_path.name}")
+            return try_path
+        except FileExistsError:
+            continue
+        except OSError as e:
+            if _is_permission_error(e) or e.errno in (errno.EXDEV, errno.EMLINK):
+                if _is_permission_error(e):
+                    log_transfer_permission_context(
+                        "atomic_hardlink",
+                        source=source_path,
+                        dest=try_path,
+                        error=e,
+                    )
+                logger.debug(
+                    "Hardlink failed (%s), falling back to copy: %s -> %s",
+                    e,
+                    source_path,
+                    dest_path,
+                )
+                return atomic_copy(source_path, dest_path, max_attempts=max_attempts)
+            raise
+
+    raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
+
+
+def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
+    """Copy a file with atomic collision detection.
+
+    Uses exclusive create to claim destination, then copies via temp file
+    to avoid partial files on failure.
+
+    Args:
+        source_path: Source file to copy
+        dest_path: Desired destination path
+        max_attempts: Maximum collision retries before raising error
+
+    Returns:
+        Path where file was actually copied (may differ from dest_path)
+
+    Raises:
+        RuntimeError: If no unique path found after max_attempts
+    """
+    base = dest_path.stem
+    ext = dest_path.suffix
+    parent = dest_path.parent
+
+    for attempt in range(max_attempts):
+        try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
+        try:
+            # Atomically claim the destination by creating an exclusive file
+            fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
+            os.close(fd)
+            
+            # Copy to temp file first, then replace to avoid partial files
+            temp_path = try_path.parent / f".{try_path.name}.tmp"
+            try:
+                try:
+                    shutil.copy2(str(source_path), str(temp_path))
+                except (PermissionError, OSError) as e:
+                    # Handle NFS permission errors immediately here
+                    if _is_permission_error(e):
+                        log_transfer_permission_context(
+                            "atomic_copy",
+                            source=source_path,
+                            dest=temp_path,
+                            error=e,
+                        )
+                        logger.debug(
+                            "Permission error during copy, falling back to copyfile (%s -> %s): %s",
+                            source_path,
+                            temp_path,
+                            e,
+                        )
+                        try:
+                            _perform_nfs_fallback(source_path, temp_path, is_move=False)
+                        except Exception as fallback_error:
+                            logger.error(
+                                "NFS fallback also failed (%s -> %s): %s",
+                                source_path,
+                                temp_path,
+                                fallback_error,
+                            )
+                            raise e from fallback_error
+                    else:
+                        raise
+                
+                temp_path.replace(try_path)
+                _verify_transfer_size(try_path, source_path.stat().st_size, "copy")
+                if attempt > 0:
+                    logger.info(f"File collision resolved: {try_path.name}")
+                return try_path
+            except Exception:
+                try_path.unlink(missing_ok=True)
+                temp_path.unlink(missing_ok=True)
+                raise
+        except FileExistsError:
+            continue
+
+    raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")

shelfmark/download/http.py

@@ -1,4 +1,4 @@
-"""Network operations manager for the book downloader application."""
+"""HTTP download with retry, resume, and Cloudflare bypass support."""
 
 import random
 import time
@@ -10,26 +10,105 @@ from urllib.parse import urlparse
 import requests
 from tqdm import tqdm
 
-import network
-from config import PROXIES
-from env import DEFAULT_SLEEP, MAX_RETRY, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER
-from logger import setup_logger
-
-# Import bypasser if enabled
-if USE_CF_BYPASS:
-    if USING_EXTERNAL_BYPASSER:
-        from cloudflare_bypasser_external import get_bypassed_page
-        # External bypasser doesn't share cookies
-        get_cf_cookies_for_domain = lambda domain: {}
-    else:
-        from cloudflare_bypasser import get_bypassed_page, get_cf_cookies_for_domain
+from shelfmark.download import network
+from shelfmark.download.network import get_proxies
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
+# Bypasser modules are imported lazily to support dynamic selection based on config
+_internal_bypasser = None
+_external_bypasser = None
+
+
+def _get_internal_bypasser():
+    """Lazy import of internal bypasser module."""
+    global _internal_bypasser
+    if _internal_bypasser is None:
+        try:
+            from shelfmark.bypass import internal_bypasser
+            _internal_bypasser = internal_bypasser
+        except ImportError as e:
+            raise RuntimeError(
+                f"Failed to import internal bypasser: {e}. "
+                "Check that all dependencies are installed. "
+                "You may need to disable CF bypass or use the external bypasser."
+            ) from e
+    return _internal_bypasser
+
+
+def _get_external_bypasser():
+    """Lazy import of external bypasser module."""
+    global _external_bypasser
+    if _external_bypasser is None:
+        try:
+            from shelfmark.bypass import external_bypasser
+            _external_bypasser = external_bypasser
+        except ImportError as e:
+            raise RuntimeError(
+                f"Failed to import external bypasser: {e}. "
+                "Check that the external bypasser is properly configured."
+            ) from e
+    return _external_bypasser
+
+
+def _is_using_external_bypasser() -> bool:
+    """Check if external bypasser is configured (reads from config, not just env)."""
+    return app_config.get("USING_EXTERNAL_BYPASSER", False)
+
+
+def _is_cf_bypass_enabled() -> bool:
+    """Check if Cloudflare bypass is enabled."""
+    return app_config.get("USE_CF_BYPASS", True)
+
+
+def get_bypassed_page(url, selector=None, cancel_flag=None):
+    """Wrapper that delegates to the appropriate bypasser based on config."""
+    if _is_using_external_bypasser():
+        return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag)
+    return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
+
+
+def get_cf_cookies_for_domain(domain):
+    """Get CF cookies - only available with internal bypasser."""
+    if _is_using_external_bypasser():
+        logger.debug(f"External bypasser in use, CF cookies not available for {domain}")
+        return {}
+    return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
+
+
+def get_cf_user_agent_for_domain(domain):
+    """Get CF user agent - only available with internal bypasser."""
+    if _is_using_external_bypasser():
+        logger.debug(f"External bypasser in use, CF user agent not available for {domain}")
+        return None
+    return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
+
+
+def _apply_cf_bypass(url: str, headers: dict) -> dict:
+    """Apply CF bypass cookies and user agent if available.
+
+    Modifies headers in-place with the stored user agent (if available).
+    Returns cookies dict to use with the request.
+    """
+    if not _is_cf_bypass_enabled():
+        return {}
+
+    parsed = urlparse(url)
+    hostname = parsed.hostname or ""
+    cookies = get_cf_cookies_for_domain(hostname)
+    stored_ua = get_cf_user_agent_for_domain(hostname)
+    if stored_ua:
+        headers['User-Agent'] = stored_ua
+    return cookies
+
+
 # Network settings
 REQUEST_TIMEOUT = (5, 10)  # (connect, read)
 MAX_DOWNLOAD_RETRIES = 2
 MAX_RESUME_ATTEMPTS = 3
+
 RETRYABLE_CODES = (429, 500, 502, 503, 504)
 CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout,
                      requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError)
@@ -37,6 +116,9 @@ DOWNLOAD_HEADERS = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
     'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
 }
 
 
@@ -70,7 +152,7 @@ def _is_retryable_error(e: Exception) -> bool:
     if isinstance(e, CONNECTION_ERRORS):
         return True
     status = _get_status_code(e)
-    return status in RETRYABLE_CODES if status else False
+    return status is not None and status in RETRYABLE_CODES
 
 
 def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]:
@@ -89,12 +171,20 @@ def _try_rotation(original_url: str, current_url: str, selector: network.AAMirro
 
 def html_get_page(
     url: str,
-    retry: int = MAX_RETRY,
+    retry: Optional[int] = None,
     use_bypasser: bool = False,
     selector: Optional[network.AAMirrorSelector] = None,
     cancel_flag: Optional[Event] = None,
+    status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
+    allow_bypasser_fallback: bool = True,
 ) -> str:
-    """Fetch HTML content from a URL with retry mechanism."""
+    """Fetch HTML content from a URL with retry mechanism.
+
+    Args:
+        allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
+            instead of switching to the bypasser. Use for search operations.
+    """
+    retry = retry if retry is not None else app_config.MAX_RETRY
     selector = selector or network.AAMirrorSelector()
     original_url = url
     current_url = selector.rewrite(original_url)
@@ -107,8 +197,10 @@ def html_get_page(
             return ""
 
         try:
-            if use_bypasser_now and USE_CF_BYPASS:
-                logger.info(f"GET (bypasser): {current_url}")
+            if use_bypasser_now and _is_cf_bypass_enabled():
+                logger.debug(f"GET (bypasser): {current_url}")
+                if status_callback:
+                    status_callback("resolving", "Bypassing protection")
                 try:
                     result = get_bypassed_page(current_url, selector, cancel_flag)
                     return result or ""
@@ -116,13 +208,11 @@ def html_get_page(
                     logger.warning(f"Bypasser error: {type(e).__name__}: {e}")
                     return ""
 
-            logger.info(f"GET: {current_url}")
-            # Try with CF cookies if available (from previous bypass)
-            cookies = {}
-            if USE_CF_BYPASS:
-                parsed = urlparse(current_url)
-                cookies = get_cf_cookies_for_domain(parsed.hostname or "")
-            response = requests.get(current_url, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies)
+            logger.debug(f"GET: {current_url}")
+            # Try with CF cookies/UA if available (from previous bypass)
+            headers = {}
+            cookies = _apply_cf_bypass(current_url, headers)
+            response = requests.get(current_url, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
             response.raise_for_status()
             time.sleep(1)
             return response.text
@@ -132,7 +222,16 @@ def html_get_page(
 
             # 403 = Cloudflare/DDoS-Guard protection
             if status == 403:
-                if USE_CF_BYPASS and not use_bypasser_now:
+                # If bypasser fallback is disabled, try mirrors instead
+                if not allow_bypasser_fallback:
+                    new_url = _try_rotation(original_url, current_url, selector)
+                    if new_url:
+                        current_url = new_url
+                        continue
+                    logger.warning(f"403 error, mirrors exhausted: {current_url}")
+                    return ""
+
+                if _is_cf_bypass_enabled() and not use_bypasser_now:
                     # Before switching to bypasser, check if cookies have become available
                     # (another concurrent download may have completed bypass and extracted cookies)
                     parsed = urlparse(current_url)
@@ -142,6 +241,8 @@ def html_get_page(
                         logger.debug(f"403 but cookies now available - retrying with cookies: {current_url}")
                         continue
                     logger.info(f"403 detected; switching to bypasser: {current_url}")
+                    if status_callback:
+                        status_callback("resolving", "Bypassing protection...")
                     use_bypasser_now = True
                     continue
                 logger.warning(f"403 error, giving up: {current_url}")
@@ -189,6 +290,7 @@ def download_url(
     total_size = parse_size_string(size) or 0
 
     attempt = 0
+    zlib_cookie_refresh_attempted = False
 
     while attempt < MAX_DOWNLOAD_RETRIES:
         if cancel_flag and cancel_flag.is_set():
@@ -202,14 +304,9 @@ def download_url(
                 status_callback("resolving", f"Connecting (Attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
 
             logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
-            # Try with CF cookies if available
-            cookies = {}
-            if USE_CF_BYPASS:
-                parsed = urlparse(current_url)
-                cookies = get_cf_cookies_for_domain(parsed.hostname or "")
-                if cookies:
-                    logger.debug(f"Using {len(cookies)} cookies for {parsed.hostname}: {list(cookies.keys())}")
-            response = requests.get(current_url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
+            # Try with CF cookies/UA if available
+            cookies = _apply_cf_bypass(current_url, headers)
+            response = requests.get(current_url, stream=True, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
             response.raise_for_status()
 
             if status_callback:
@@ -243,6 +340,20 @@ def download_url(
             status = _get_status_code(e)
             retryable = _is_retryable_error(e)
 
+            # Z-Library 403 - try refreshing cookies via bypasser once before giving up
+            if status == 403 and _is_cf_bypass_enabled() and not zlib_cookie_refresh_attempted:
+                parsed = urlparse(current_url)
+                if parsed.hostname and 'z-lib' in parsed.hostname and referer:
+                    zlib_cookie_refresh_attempted = True
+                    logger.info(f"Z-Library 403 - refreshing cookies via referer: {referer}")
+                    try:
+                        get_bypassed_page(referer, selector, cancel_flag)
+                        time.sleep(0.5)
+                        # Retry with fresh cookies (don't increment attempt)
+                        continue
+                    except Exception as cookie_err:
+                        logger.warning(f"Z-Library cookie refresh failed: {cookie_err}")
+
             # Non-retryable errors
             if status in (403, 404):
                 logger.warning(f"Download failed ({status}): {current_url}")
@@ -253,14 +364,14 @@ def download_url(
             if status == 429:
                 logger.info(f"Rate limited (429) - trying next source")
                 if status_callback:
-                    status_callback("resolving", "Server busy, trying next...")
+                    status_callback("resolving", "Server busy, trying next")
                 return None
 
             # Timeout - don't retry, server likely overloaded
             if isinstance(e, requests.exceptions.Timeout):
                 logger.warning(f"Timeout: {current_url} - skipping to next source")
                 if status_callback:
-                    status_callback("resolving", "Server timed out, trying next...")
+                    status_callback("resolving", "Server timed out, trying next")
                 return None
 
             # Try to resume if we got some data
@@ -301,14 +412,11 @@ def _try_resume(
         time.sleep(_backoff_delay(attempt + 1, base=0.5, cap=5.0))
 
         try:
-            # Try with CF cookies if available
-            cookies = {}
-            if USE_CF_BYPASS:
-                parsed = urlparse(url)
-                cookies = get_cf_cookies_for_domain(parsed.hostname or "")
+            # Try with CF cookies/UA if available
             resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'}
+            cookies = _apply_cf_bypass(url, resume_headers)
             response = requests.get(
-                url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT,
+                url, stream=True, proxies=get_proxies(url), timeout=REQUEST_TIMEOUT,
                 headers=resume_headers, cookies=cookies
             )
             

shelfmark/download/network.py

@@ -1,5 +1,6 @@
-"""Network operations manager for the book downloader application."""
+"""DNS rotation, mirror selection, and network utilities."""
 
+import fnmatch
 import requests
 import urllib.request
 from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable
@@ -7,15 +8,93 @@ import socket
 import dns.resolver
 from socket import AddressFamily, SocketKind
 import urllib.parse
-import ssl
 import ipaddress
 
-from logger import setup_logger
-from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER
-import config
-import env
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.config import config as app_config
+from shelfmark.core.utils import normalize_http_url
 from datetime import datetime, timedelta
 
+
+def _get_no_proxy_patterns() -> List[str]:
+    """Get list of NO_PROXY patterns from config."""
+    no_proxy = app_config.get("NO_PROXY", "")
+    if not no_proxy:
+        return []
+    return [p.strip().lower() for p in no_proxy.split(",") if p.strip()]
+
+
+def should_bypass_proxy(url: str) -> bool:
+    """Check if a URL should bypass the proxy based on NO_PROXY patterns.
+
+    Supports:
+    - Exact hostname match: localhost, myhost.local
+    - Wildcard prefix: *.local matches foo.local
+    - Wildcard suffix: 10.* matches 10.1.2.3
+    """
+    if not url:
+        return False
+
+    patterns = _get_no_proxy_patterns()
+    if not patterns:
+        return False
+
+    # Extract hostname from URL
+    try:
+        parsed = urllib.parse.urlparse(url)
+        hostname = (parsed.hostname or "").lower()
+    except Exception as e:
+        logger.debug(f"Failed to parse URL for proxy bypass check: {url} - {e}")
+        return False
+
+    if not hostname:
+        return False
+
+    for pattern in patterns:
+        # Use fnmatch for wildcard matching (supports * and ?)
+        if fnmatch.fnmatch(hostname, pattern):
+            return True
+
+    return False
+
+
+def get_proxies(url: str = "") -> dict:
+    """Get current proxy configuration from config singleton.
+
+    Args:
+        url: Optional URL to check against NO_PROXY patterns.
+             If provided and matches a pattern, returns empty dict.
+    """
+    # Check NO_PROXY bypass first
+    if url and should_bypass_proxy(url):
+        return {}
+
+    proxy_mode = app_config.get("PROXY_MODE", "none")
+
+    if proxy_mode == "socks5":
+        socks_proxy = app_config.get("SOCKS5_PROXY", "")
+        if socks_proxy:
+            return {"http": socks_proxy, "https": socks_proxy}
+    elif proxy_mode == "http":
+        proxies = {}
+        http_proxy = app_config.get("HTTP_PROXY", "")
+        https_proxy = app_config.get("HTTPS_PROXY", "")
+        if http_proxy:
+            proxies["http"] = http_proxy
+        if https_proxy:
+            proxies["https"] = https_proxy
+        elif http_proxy:
+            # Fallback: use HTTP proxy for HTTPS if HTTPS proxy not specified
+            proxies["https"] = http_proxy
+        return proxies
+
+    return {}
+
+# DNS state - authoritative values managed by this module
+# Other modules should use get_dns_config() to read these
+CUSTOM_DNS: List[str] = []
+DOH_SERVER: str = ""
+
 # Try to use gevent locks if available (for gevent worker compatibility)
 # Fall back to threading locks for non-gevent environments
 try:
@@ -77,13 +156,6 @@ def _notify_dns_rotation(provider_name: str, servers: List[str], doh_url: str) -
         except Exception as e:
             logger.warning(f"DNS rotation callback {callback.__name__} failed: {e}")
 
-def _agent_debug_log(code: str, source: str, reason: str, meta: Optional[dict] = None) -> None:
-    """Lightweight debug hook for automated runs; safe no-op on failure."""
-    try:
-        logger.debug(f"[agent] code={code} source={source} reason={reason} meta={meta or {}}")
-    except Exception as exc:
-        # Avoid raising inside debug logger
-        logger.debug(f"[agent] log failed: {exc}")
 
 def _load_state():
     """Return current in-memory network state (no disk persistence)."""
@@ -103,7 +175,8 @@ def _save_state(aa_url=None, dns_provider=None):
 
 # AA URL failover state
 _current_aa_url_index = 0
-_aa_urls = AA_AVAILABLE_URLS.copy()
+_aa_urls: List[str] = []  # Initialized lazily in _initialize_aa_state()
+_aa_base_url: str = ""  # Current active AA URL
 
 def _ensure_initialized() -> None:
     """Lazy guard so runtime setup happens once and late calls still work."""
@@ -143,7 +216,9 @@ _dns_exhausted_logged = False
 
 def _is_auto_dns_mode() -> bool:
     """Check if DNS is in auto-rotation mode."""
-    return env._CUSTOM_DNS.lower().strip() == "auto" and not env.USING_TOR
+    custom_dns = app_config.get("CUSTOM_DNS", "auto")
+    using_tor = app_config.get("USING_TOR", False)
+    return str(custom_dns).lower().strip() == "auto" and not using_tor
 
 
 def _current_dns_label() -> str:
@@ -151,9 +226,44 @@ def _current_dns_label() -> str:
     if _current_dns_index >= 0:
         return DNS_PROVIDERS[_current_dns_index][0]
     if CUSTOM_DNS:
-        return f"custom {CUSTOM_DNS}"
+        return f"manual ({len(CUSTOM_DNS)} servers)"
     return "system"
 
+
+def get_dns_config() -> dict:
+    """
+    Get the current DNS configuration.
+
+    Returns:
+        Dict with keys:
+        - provider: str - Current provider name ('auto', 'system', 'google', 'cloudflare', etc.)
+        - servers: List[str] - DNS server IPs in use
+        - doh_url: str - DoH server URL (empty if disabled)
+        - doh_enabled: bool - Whether DoH is active
+        - is_auto_mode: bool - Whether auto-rotation is enabled
+    """
+    _ensure_initialized()
+
+    custom_dns = str(app_config.get("CUSTOM_DNS", "auto")).lower().strip()
+    if _current_dns_index >= 0:
+        provider = DNS_PROVIDERS[_current_dns_index][0]
+    elif custom_dns == "auto":
+        provider = "auto"
+    elif custom_dns == "system":
+        provider = "system"
+    elif custom_dns == "manual":
+        provider = "manual"
+    else:
+        provider = custom_dns
+
+    return {
+        "provider": provider,
+        "servers": list(CUSTOM_DNS),
+        "doh_url": DOH_SERVER,
+        "doh_enabled": bool(DOH_SERVER),
+        "is_auto_mode": _is_auto_dns_mode(),
+    }
+
 # Common helper functions for DNS resolution
 def _decode_host(host: Union[str, bytes, None]) -> str:
     """Convert host to string, handling bytes and None cases."""
@@ -167,34 +277,41 @@ def _decode_port(port: Union[str, bytes, int, None]) -> int:
     """Convert port to integer, handling various input types."""
     if port is None:
         return 0
-    if isinstance(port, (str, bytes)):
-        return int(port)
     return int(port)
 
 def _is_local_address(host_str: str) -> bool:
-    """Check if an address is local or private and should bypass custom DNS."""
-    # Localhost checks
-    if (host_str == 'localhost' or 
-        host_str.startswith('127.') or 
-        host_str == '::1' or 
-        host_str == '0.0.0.0'):
+    """Check if an address is local/private and should bypass custom DNS.
+
+    Returns True for:
+    - 'localhost'
+    - Private/loopback/link-local IP addresses
+    - Simple hostnames without a dot (e.g., 'booklore', 'prowlarr') - likely Docker service names
+    - Hostnames ending in common internal TLDs (.local, .internal, .lan, .home, .docker)
+    """
+    if not host_str:
+        return False
+
+    host_lower = host_str.lower()
+
+    # Check for localhost
+    if host_lower == 'localhost':
         return True
-        
-    # IPv4 private ranges (RFC 1918)
-    if (host_str.startswith('10.') or 
-        (host_str.startswith('172.') and 
-         len(host_str.split('.')) > 1 and 
-         16 <= int(host_str.split('.')[1]) <= 31) or
-        host_str.startswith('192.168.')):
+
+    # Check for simple hostnames (no dot = likely internal Docker/container name)
+    if '.' not in host_str:
         return True
-        
-    # IPv6 private ranges
-    if (host_str.startswith('fc') or 
-        host_str.startswith('fd') or  # Unique local addresses (fc00::/7)
-        host_str.startswith('fe80:')):  # Link-local addresses (fe80::/10)
+
+    # Check for common internal TLDs
+    internal_tlds = ('.local', '.internal', '.lan', '.home', '.docker', '.localdomain')
+    if any(host_lower.endswith(tld) for tld in internal_tlds):
         return True
-    
-    return False
+
+    # Check for private/loopback/link-local IP addresses
+    try:
+        addr = ipaddress.ip_address(host_str)
+        return addr.is_private or addr.is_loopback or addr.is_link_local
+    except ValueError:
+        return False
 
 def _is_ip_address(host_str: str) -> bool:
     """Check if a string is a valid IP address (IPv4 or IPv6)."""
@@ -300,7 +417,7 @@ class DoHResolver:
             response = self.session.get(
                 self.base_url,
                 params=params,
-                proxies=PROXIES,
+                proxies=get_proxies(self.base_url),
                 timeout=10  # Increased from 5s to handle slow network conditions
             )
             response.raise_for_status()
@@ -575,8 +692,8 @@ def switch_dns_provider() -> bool:
         name, servers, doh = DNS_PROVIDERS[_current_dns_index]
         CUSTOM_DNS = servers
         DOH_SERVER = doh
-        config.CUSTOM_DNS = servers
-        config.DOH_SERVER = doh
+        app_config.CUSTOM_DNS = servers
+        app_config.DOH_SERVER = doh
 
         logger.warning(f"Switched DNS provider to: {name} (using DoH)")
         _save_state(dns_provider=name)
@@ -590,37 +707,128 @@ def switch_dns_provider() -> bool:
 def rotate_dns_provider() -> bool:
     """Rotate DNS provider (auto mode only), cycling back if exhausted."""
     global _current_dns_index, _dns_exhausted_logged
-    
+
     if not _is_auto_dns_mode():
         return False
-    
+
     if _current_dns_index + 1 >= len(DNS_PROVIDERS):
         logger.warning("DNS rotation: cycling back to first provider")
         _current_dns_index = -1
         _dns_exhausted_logged = False
-    
+
     return switch_dns_provider()
 
 def rotate_dns_and_reset_aa() -> bool:
     """
     Switch DNS provider (auto mode) and reset AA URL list to the first entry.
     Returns True if DNS switched; False if no providers left or not in auto mode.
-    
+
     Note: This function can be called during initialization, so we must NOT call
     _ensure_initialized() here to avoid recursive init loops.
     """
     if not rotate_dns_provider():
         return False
     # Reset AA URL to first available auto option if using auto AA
-    global AA_BASE_URL, _current_aa_url_index
-    if AA_BASE_URL == "auto" or AA_BASE_URL in _aa_urls:
+    global _aa_base_url, _current_aa_url_index
+    configured_url = normalize_http_url(
+        app_config.get("AA_BASE_URL", "auto"),
+        default_scheme="https",
+        allow_special=("auto",),
+    )
+    if not configured_url:
+        configured_url = "auto"
+
+    if configured_url == "auto" or _aa_base_url in _aa_urls:
         _current_aa_url_index = 0
-        AA_BASE_URL = _aa_urls[0]
-        config.AA_BASE_URL = AA_BASE_URL
-        logger.info(f"After DNS switch, resetting AA URL to: {AA_BASE_URL}")
-        _save_state(aa_url=AA_BASE_URL)
+        _aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.se"
+        logger.info(f"After DNS switch, resetting AA URL to: {_aa_base_url}")
+        _save_state(aa_url=_aa_base_url)
     return True
 
+def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use_doh: bool | None = None) -> bool:
+    """
+    Set DNS to a specific provider or manual servers.
+
+    Args:
+        provider: One of 'auto', 'system', 'google', 'cloudflare', 'quad9', 'opendns', 'manual'
+        manual_servers: List of DNS server IPs when provider is 'manual'
+        use_doh: Whether to use DNS over HTTPS. If None, uses current USE_DOH config setting.
+                 Note: Auto mode always uses DoH for reliability during rotation.
+
+    Returns:
+        True if DNS was changed successfully.
+    """
+    global CUSTOM_DNS, DOH_SERVER, _current_dns_index, _dns_exhausted_logged
+
+    provider = provider.lower().strip()
+
+    # Determine DoH preference - use provided value or fall back to config setting
+    doh_enabled = use_doh if use_doh is not None else app_config.get("USE_DOH", True)
+
+    with _dns_switch_lock:
+        if provider == "system":
+            # Use system DNS only - no custom resolver, no failover rotation
+            _current_dns_index = -1
+            _dns_exhausted_logged = False
+            CUSTOM_DNS = []
+            DOH_SERVER = ""
+            app_config.CUSTOM_DNS = []
+            app_config.DOH_SERVER = ""
+            # Restore original system getaddrinfo
+            socket.getaddrinfo = original_getaddrinfo
+            logger.info("DNS set to system mode (using OS default resolver)")
+            _notify_dns_rotation("system", [], "")
+            return True
+
+        if provider == "auto":
+            # Reset to auto mode - start with system DNS
+            # Note: Auto mode always uses DoH when rotating for reliability
+            _current_dns_index = -1
+            _dns_exhausted_logged = False
+            CUSTOM_DNS = []
+            DOH_SERVER = ""
+            app_config.CUSTOM_DNS = []
+            app_config.DOH_SERVER = ""
+            logger.info("DNS set to auto mode (system DNS, will rotate on failure with DoH)")
+            init_dns_resolvers()
+            _notify_dns_rotation("auto", [], "")
+            return True
+
+        if provider == "manual":
+            if not manual_servers:
+                logger.warning("Manual DNS requested but no servers provided")
+                return False
+            _current_dns_index = -1  # Not using preset providers
+            CUSTOM_DNS = manual_servers
+            DOH_SERVER = ""  # No DoH for manual servers
+            app_config.CUSTOM_DNS = manual_servers
+            app_config.DOH_SERVER = ""
+            logger.info(f"DNS set to manual servers: {manual_servers}")
+            init_dns_resolvers()
+            _notify_dns_rotation("manual", manual_servers, "")
+            return True
+
+        # Find the provider in DNS_PROVIDERS
+        for i, (name, servers, doh) in enumerate(DNS_PROVIDERS):
+            if name == provider:
+                _current_dns_index = i
+                _dns_exhausted_logged = False
+                CUSTOM_DNS = servers
+                # Only set DoH server if DoH is enabled
+                DOH_SERVER = doh if doh_enabled else ""
+                app_config.CUSTOM_DNS = servers
+                app_config.DOH_SERVER = DOH_SERVER
+                doh_status = "DoH enabled" if doh_enabled else "standard DNS"
+                logger.info(f"DNS set to: {name} ({doh_status})")
+                _save_state(dns_provider=name)
+                init_dns_resolvers()
+                _notify_dns_rotation(name, servers, DOH_SERVER)
+                return True
+
+        logger.warning(f"Unknown DNS provider: {provider}")
+        return False
+
+
 def init_dns_resolvers():
     """Initialize DNS resolvers based on configuration."""
     global CUSTOM_DNS, DOH_SERVER
@@ -630,15 +838,15 @@ def init_dns_resolvers():
             name, servers, doh = DNS_PROVIDERS[_current_dns_index]
             CUSTOM_DNS = servers
             DOH_SERVER = doh
-            config.CUSTOM_DNS = servers
-            config.DOH_SERVER = doh
+            app_config.CUSTOM_DNS = servers
+            app_config.DOH_SERVER = doh
             logger.info(f"Using DNS provider: {name} (DoH enabled)")
         else:
             CUSTOM_DNS = []
             DOH_SERVER = ""
-            config.CUSTOM_DNS = []
-            config.DOH_SERVER = ""
-            logger.info("Using system DNS (auto mode - will switch on failure)")
+            app_config.CUSTOM_DNS = []
+            app_config.DOH_SERVER = ""
+            logger.debug("Using system DNS (auto mode - will switch on failure)")
             socket.getaddrinfo = cast(Any, create_system_failover_getaddrinfo())
             return
     
@@ -648,68 +856,130 @@ def init_dns_resolvers():
             init_doh_resolver(DOH_SERVER)
 
 
-def _initialize_dns_state() -> None:
-    """Restore persisted DNS choice or start fresh."""
-    global _current_dns_index
-    
-    if _is_auto_dns_mode():
-        persisted = state.get('dns_provider') if state else None
-        if persisted:
-            for i, (name, _, _) in enumerate(DNS_PROVIDERS):
-                if name == persisted:
-                    _current_dns_index = i
-                    logger.info(f"Restored DNS provider from state: {name}")
-                    return
-        _current_dns_index = -1
+def _get_initial_dns_config() -> tuple[str, List[str] | None, bool]:
+    """
+    Determine initial DNS configuration from config singleton.
+
+    The config singleton already handles ENV > config file > default priority,
+    so we just read from config.
+
+    Returns:
+        Tuple of (provider, manual_servers, use_doh)
+    """
+    provider = str(app_config.get("CUSTOM_DNS", "auto")).lower().strip()
+    use_doh = app_config.get("USE_DOH", True)
+    manual_servers = None
+
+    # Check for manual DNS servers in config
+    if provider == "manual":
+        manual_dns = str(app_config.get("CUSTOM_DNS_MANUAL", "")).strip()
+        if manual_dns:
+            manual_servers = [s.strip() for s in manual_dns.split(",") if s.strip()]
+
+    # Handle legacy format: IPs directly in CUSTOM_DNS setting
+    if provider and provider not in ("auto", "system", "google", "cloudflare", "quad9", "opendns", "manual", ""):
+        # Check if it looks like IP addresses
+        parts = provider.split(",")
+        potential_ips = [p.strip() for p in parts if p.strip()]
+        if potential_ips and all(_looks_like_ip(p) for p in potential_ips):
+            manual_servers = potential_ips
+            provider = "manual"
+            logger.info(f"Detected legacy DNS format, treating as manual: {manual_servers}")
+
+    return provider or "auto", manual_servers, use_doh
+
+
+def _looks_like_ip(s: str) -> bool:
+    """Check if a string looks like an IP address."""
+    # Simple heuristic: contains only digits, dots, and colons
+    return s.replace(".", "").replace(":", "").isdigit()
+
+def _build_aa_urls() -> List[str]:
+    """Build list of available AA URLs from centralized mirror config."""
+    from shelfmark.core.mirrors import get_aa_mirrors
+    return get_aa_mirrors()
+
 
 def _initialize_aa_state() -> None:
     """Restore or probe AA URL state."""
-    global AA_BASE_URL, _current_aa_url_index
-    if AA_BASE_URL == "auto":
+    global _aa_base_url, _current_aa_url_index, _aa_urls
+
+    # Build URL list from config
+    _aa_urls = _build_aa_urls()
+
+    # Get configured base URL from config
+    configured_url = normalize_http_url(
+        app_config.get("AA_BASE_URL", "auto"),
+        default_scheme="https",
+        allow_special=("auto",),
+    )
+    if not configured_url:
+        configured_url = "auto"
+
+    if configured_url == "auto":
         if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls:
             _current_aa_url_index = _aa_urls.index(state['aa_base_url'])
-            AA_BASE_URL = state['aa_base_url']
+            _aa_base_url = state['aa_base_url']
         else:
-            logger.info(f"AA_BASE_URL: auto, checking available urls {_aa_urls}")
+            logger.debug(f"AA_BASE_URL: auto, checking available urls {_aa_urls}")
             for i, url in enumerate(_aa_urls):
                 try:
-                    response = requests.get(url, proxies=PROXIES, timeout=3)
+                    response = requests.get(url, proxies=get_proxies(url), timeout=3)
                     if response.status_code == 200:
                         _current_aa_url_index = i
-                        AA_BASE_URL = url
-                        _save_state(aa_url=AA_BASE_URL)
+                        _aa_base_url = url
+                        _save_state(aa_url=_aa_base_url)
                         break
                 except Exception:
                     pass
-            if AA_BASE_URL == "auto":
-                AA_BASE_URL = _aa_urls[0]
+            if not _aa_base_url or _aa_base_url == "auto":
+                _aa_base_url = _aa_urls[0]
                 _current_aa_url_index = 0
-    elif AA_BASE_URL not in _aa_urls:
-        logger.info(f"AA_BASE_URL set to custom value {AA_BASE_URL}; skipping auto-switch")
+    elif configured_url not in _aa_urls:
+        logger.info(f"AA_BASE_URL set to custom value {configured_url}; skipping auto-switch")
+        _aa_base_url = configured_url
     else:
-        _current_aa_url_index = _aa_urls.index(AA_BASE_URL)
+        _current_aa_url_index = _aa_urls.index(configured_url)
+        _aa_base_url = configured_url
 
-    config.AA_BASE_URL = AA_BASE_URL
-    logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
+    logger.info(f"AA_BASE_URL: {_aa_base_url}")
 
 def init_dns(force: bool = False) -> None:
-    """Initialize DNS state and resolvers."""
-    global state, _dns_initialized
+    """Initialize DNS state and resolvers using set_dns_provider() for consistency."""
+    global state, _dns_initialized, _current_dns_index
     if _dns_initialized and not force:
         return
     with _init_lock:
         # Double-check after acquiring lock
         if _dns_initialized and not force:
             return
-        # Set flag BEFORE doing work to prevent recursive calls during init
-        _dns_initialized = True
+        # Do work first, set flag after to prevent race conditions
         try:
             logger.debug(f"Initializing DNS (using {'gevent' if _using_gevent_locks else 'threading'} locks)")
             state = _load_state()
-            _initialize_dns_state()
-            init_dns_resolvers()
+
+            # Get initial DNS configuration from environment
+            provider, manual_servers, use_doh = _get_initial_dns_config()
+
+            if provider == "auto":
+                # Auto mode: check for persisted provider from previous rotation
+                persisted = state.get('dns_provider') if state else None
+                if persisted:
+                    for i, (name, _, _) in enumerate(DNS_PROVIDERS):
+                        if name == persisted:
+                            _current_dns_index = i
+                            logger.info(f"Restored DNS provider from state: {name}")
+                            break
+                # Use init_dns_resolvers() for auto mode to preserve rotation capability
+                init_dns_resolvers()
+            else:
+                # Non-auto mode: use set_dns_provider() for consistent initialization
+                set_dns_provider(provider, manual_servers, use_doh=use_doh)
+
+            # Only set flag AFTER work completes successfully
+            _dns_initialized = True
         except Exception:
-            _dns_initialized = False
+            # Flag stays False so retry is possible
             raise
 
 def init_aa(force: bool = False) -> None:
@@ -721,13 +991,14 @@ def init_aa(force: bool = False) -> None:
         # Double-check after acquiring lock
         if _aa_initialized and not force:
             return
-        # Set flag BEFORE doing work to prevent recursive calls during init
-        _aa_initialized = True
+        # Do work first, set flag after to prevent race conditions
         try:
             state = _load_state()
             _initialize_aa_state()
+            # Only set flag AFTER work completes successfully
+            _aa_initialized = True
         except Exception:
-            _aa_initialized = False
+            # Flag stays False so retry is possible
             raise
 
 def init(force: bool = False) -> None:
@@ -744,21 +1015,21 @@ def init(force: bool = False) -> None:
         # Double-check after acquiring lock
         if _initialized and not force:
             return
-        # Set flag BEFORE doing work to prevent recursive calls during init
-        # (e.g., DNS failover handlers calling back into init)
-        _initialized = True
+        # Do the work first, then set flag to prevent race conditions
+        # where another thread sees _initialized=True but _aa_base_url is still empty
         try:
             init_dns(force=force)
             init_aa(force=force)
+            # Only set flag AFTER work completes successfully
+            _initialized = True
         except Exception:
-            # Reset flag on failure so retry is possible
-            _initialized = False
+            # Flag stays False so retry is possible
             raise
 
 def get_aa_base_url():
     """Get current AA base URL."""
     _ensure_initialized()
-    return AA_BASE_URL
+    return _aa_base_url
 
 def get_available_aa_urls():
     """Get list of configured AA URLs (copy)."""
@@ -768,14 +1039,13 @@ def get_available_aa_urls():
 def set_aa_url_index(new_index: int) -> bool:
     """Set AA base URL by index in available list; returns True if applied."""
     _ensure_initialized()
-    global AA_BASE_URL, _current_aa_url_index
+    global _aa_base_url, _current_aa_url_index
     if new_index < 0 or new_index >= len(_aa_urls):
         return False
     _current_aa_url_index = new_index
-    AA_BASE_URL = _aa_urls[_current_aa_url_index]
-    config.AA_BASE_URL = AA_BASE_URL
-    logger.info(f"Set AA URL to: {AA_BASE_URL}")
-    _save_state(aa_url=AA_BASE_URL)
+    _aa_base_url = _aa_urls[_current_aa_url_index]
+    logger.info(f"Set AA URL to: {_aa_base_url}")
+    _save_state(aa_url=_aa_base_url)
     return True
 
 class AAMirrorSelector:

shelfmark/download/orchestrator.py

@@ -0,0 +1,575 @@
+"""Download queue orchestration and worker management.
+
+Two-stage architecture: handlers stage to TMP_DIR, orchestrator moves to INGEST_DIR
+with archive extraction and custom script support.
+"""
+
+import os
+import random
+import threading
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from threading import Event, Lock
+from typing import Any, Dict, List, Optional, Tuple
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters, SearchMode
+from shelfmark.core.queue import book_queue
+from shelfmark.core.utils import transform_cover_url
+from shelfmark.download.postprocess.pipeline import is_torrent_source, safe_cleanup_path
+from shelfmark.download.postprocess.router import post_process_download
+from shelfmark.release_sources import direct_download, get_handler, get_source_display_name
+from shelfmark.release_sources.direct_download import SearchUnavailable
+
+logger = setup_logger(__name__)
+
+
+# =============================================================================
+# Task Download and Processing
+# =============================================================================
+#
+# Post-download processing (staging, extraction, transfers, cleanup) lives in
+# `shelfmark.download.postprocess`.
+
+
+# WebSocket manager (initialized by app.py)
+# Track whether WebSocket is available for status reporting
+WEBSOCKET_AVAILABLE = True
+try:
+    from shelfmark.api.websocket import ws_manager
+except ImportError:
+    logger.error("WebSocket unavailable - real-time updates disabled")
+    ws_manager = None
+    WEBSOCKET_AVAILABLE = False
+
+# Progress update throttling - track last broadcast time per book
+_progress_last_broadcast: Dict[str, float] = {}
+_progress_lock = Lock()
+
+# Stall detection - track last activity time per download
+_last_activity: Dict[str, float] = {}
+STALL_TIMEOUT = 300  # 5 minutes without progress/status update = stalled
+
+def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
+    """Search for books matching the query."""
+    try:
+        books = direct_download.search_books(query, filters)
+        return [_book_info_to_dict(book) for book in books]
+    except SearchUnavailable:
+        raise
+    except Exception as e:
+        logger.error_trace(f"Error searching books: {e}")
+        raise
+
+def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
+    """Get detailed information for a specific book."""
+    try:
+        book = direct_download.get_book_info(book_id)
+        return _book_info_to_dict(book)
+    except Exception as e:
+        logger.error_trace(f"Error getting book info: {e}")
+        raise
+
+def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> Tuple[bool, Optional[str]]:
+    """Add a book to the download queue. Returns (success, error_message)."""
+    try:
+        book_info = direct_download.get_book_info(book_id, fetch_download_count=False)
+        if not book_info:
+            error_msg = f"Could not fetch book info for {book_id}"
+            logger.warning(error_msg)
+            return False, error_msg
+
+        # Create a source-agnostic download task
+        task = DownloadTask(
+            task_id=book_id,
+            source=source,
+            title=book_info.title,
+            author=book_info.author,
+            format=book_info.format,
+            size=book_info.size,
+            preview=book_info.preview,
+            content_type=book_info.content,
+            search_mode=SearchMode.DIRECT,
+            priority=priority,
+        )
+
+        if not book_queue.add(task):
+            logger.info(f"Book already in queue: {book_info.title}")
+            return False, "Book is already in the download queue"
+
+        logger.info(f"Book queued with priority {priority}: {book_info.title}")
+
+        # Broadcast status update via WebSocket
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+        return True, None
+    except SearchUnavailable as e:
+        error_msg = f"Search service unavailable: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except Exception as e:
+        error_msg = f"Error queueing book: {e}"
+        logger.error_trace(error_msg)
+        return False, error_msg
+
+
+def queue_release(release_data: dict, priority: int = 0) -> Tuple[bool, Optional[str]]:
+    """Add a release to the download queue. Returns (success, error_message)."""
+    try:
+        source = release_data.get('source', 'direct_download')
+        extra = release_data.get('extra', {})
+
+        # Get author, year, preview, and content_type from top-level (preferred) or extra (fallback)
+        author = release_data.get('author') or extra.get('author')
+        year = release_data.get('year') or extra.get('year')
+        preview = release_data.get('preview') or extra.get('preview')
+        content_type = release_data.get('content_type') or extra.get('content_type')
+
+        # Get series info for library naming templates
+        series_name = release_data.get('series_name') or extra.get('series_name')
+        series_position = release_data.get('series_position') or extra.get('series_position')
+        subtitle = release_data.get('subtitle') or extra.get('subtitle')
+
+        # Create a source-agnostic download task from release data
+        task = DownloadTask(
+            task_id=release_data['source_id'],
+            source=source,
+            title=release_data.get('title', 'Unknown'),
+            author=author,
+            year=year,
+            format=release_data.get('format'),
+            size=release_data.get('size'),
+            preview=preview,
+            content_type=content_type,
+            series_name=series_name,
+            series_position=series_position,
+            subtitle=subtitle,
+            search_mode=SearchMode.UNIVERSAL,
+            priority=priority,
+        )
+
+        if not book_queue.add(task):
+            logger.info(f"Release already in queue: {task.title}")
+            return False, "Release is already in the download queue"
+
+        logger.info(f"Release queued with priority {priority}: {task.title}")
+
+        # Broadcast status update via WebSocket
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+        return True, None
+
+    except ValueError as e:
+        # Handler not found for this source
+        error_msg = f"Unknown release source: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except KeyError as e:
+        error_msg = f"Missing required field in release data: {e}"
+        logger.warning(error_msg)
+        return False, error_msg
+    except Exception as e:
+        error_msg = f"Error queueing release: {e}"
+        logger.error_trace(error_msg)
+        return False, error_msg
+
+def queue_status() -> Dict[str, Dict[str, Any]]:
+    """Get current status of the download queue."""
+    status = book_queue.get_status()
+    for _, tasks in status.items():
+        for _, task in tasks.items():
+            if task.download_path and not os.path.exists(task.download_path):
+                task.download_path = None
+
+    # Convert Enum keys to strings and DownloadTask objects to dicts for JSON serialization
+    return {
+        status_type.value: {
+            task_id: _task_to_dict(task)
+            for task_id, task in tasks.items()
+        }
+        for status_type, tasks in status.items()
+    }
+
+def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]]:
+    """Get downloaded file data for a specific task."""
+    task = None
+    try:
+        task = book_queue.get_task(task_id)
+        if not task:
+            return None, None
+
+        path = task.download_path
+        if not path:
+            return None, task
+
+        with open(path, "rb") as f:
+            return f.read(), task
+    except Exception as e:
+        logger.error_trace(f"Error getting book data: {e}")
+        if task:
+            task.download_path = None
+        return None, task
+
+def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
+    """Convert BookInfo to dict, transforming cover URLs for caching."""
+    result = {
+        key: value for key, value in book.__dict__.items()
+        if value is not None
+    }
+
+    # Transform external preview URLs to local proxy URLs
+    if result.get('preview'):
+        result['preview'] = transform_cover_url(result['preview'], book.id)
+
+    return result
+
+
+def _task_to_dict(task: DownloadTask) -> Dict[str, Any]:
+    """Convert DownloadTask to dict for frontend, transforming cover URLs."""
+    # Transform external preview URLs to local proxy URLs
+    preview = transform_cover_url(task.preview, task.task_id)
+
+    return {
+        'id': task.task_id,
+        'title': task.title,
+        'author': task.author,
+        'format': task.format,
+        'size': task.size,
+        'preview': preview,
+        'content_type': task.content_type,
+        'source': task.source,
+        'source_display_name': get_source_display_name(task.source),
+        'priority': task.priority,
+        'added_time': task.added_time,
+        'progress': task.progress,
+        'status': task.status,
+        'status_message': task.status_message,
+        'download_path': task.download_path,
+    }
+
+
+def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
+    """Download a task via appropriate handler, then post-process to ingest."""
+    try:
+        # Check for cancellation before starting
+        if cancel_flag.is_set():
+            logger.info("Task %s: cancelled before starting", task_id)
+            return None
+
+        task = book_queue.get_task(task_id)
+        if not task:
+            logger.error("Task not found in queue: %s", task_id)
+            return None
+
+        title_label = task.title or "Unknown title"
+        logger.info(
+            "Task %s: starting download (%s) - %s",
+            task_id,
+            get_source_display_name(task.source),
+            title_label,
+        )
+
+        def progress_callback(progress: float) -> None:
+            update_download_progress(task_id, progress)
+
+        def status_callback(status: str, message: Optional[str] = None) -> None:
+            update_download_status(task_id, status, message)
+
+        # Get the download handler based on the task's source
+        handler = get_handler(task.source)
+        temp_path = handler.download(
+            task,
+            cancel_flag,
+            progress_callback,
+            status_callback
+        )
+
+        # Handler returns temp path - orchestrator handles post-processing
+        if not temp_path:
+            return None
+
+        temp_file = Path(temp_path)
+        if not temp_file.exists():
+            logger.error(f"Handler returned non-existent path: {temp_path}")
+            return None
+
+        # Check cancellation before post-processing
+        if cancel_flag.is_set():
+            logger.info("Task %s: cancelled before post-processing", task_id)
+            if not is_torrent_source(temp_file, task):
+                safe_cleanup_path(temp_file, task)
+            return None
+
+        logger.info("Task %s: download finished; starting post-processing", task_id)
+        logger.debug("Task %s: post-processing input path: %s", task_id, temp_file)
+
+        # Post-processing: output routing + file processing pipeline
+        result = post_process_download(temp_file, task, cancel_flag, status_callback)
+
+        if cancel_flag.is_set():
+            logger.info("Task %s: post-processing cancelled", task_id)
+        elif result:
+            logger.info("Task %s: post-processing complete", task_id)
+            logger.debug("Task %s: post-processing result: %s", task_id, result)
+        else:
+            logger.warning("Task %s: post-processing failed", task_id)
+
+        try:
+            handler.post_process_cleanup(task, success=bool(result))
+        except Exception as e:
+            logger.warning("Post-processing cleanup hook failed for %s: %s", task_id, e)
+
+        return result
+
+    except Exception as e:
+        if cancel_flag.is_set():
+            logger.info("Task %s: cancelled during error handling", task_id)
+        else:
+            logger.error_trace("Task %s: error downloading: %s", task_id, e)
+            # Update task status so user sees the failure
+            task = book_queue.get_task(task_id)
+            if task:
+                book_queue.update_status(task_id, QueueStatus.ERROR)
+                # Check for known misconfiguration from earlier versions
+                if isinstance(e, PermissionError) and "/cwa-book-ingest" in str(e):
+                    book_queue.update_status_message(
+                        task_id,
+                        "Destination misconfigured. Go to Settings → Downloads to update."
+                    )
+                else:
+                    if isinstance(e, PermissionError):
+                        book_queue.update_status_message(task_id, f"Permission denied: {e}")
+                    else:
+                        book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}")
+        return None
+
+
+
+def update_download_progress(book_id: str, progress: float) -> None:
+    """Update download progress with throttled WebSocket broadcasts."""
+    book_queue.update_progress(book_id, progress)
+
+    # Track activity for stall detection
+    with _progress_lock:
+        _last_activity[book_id] = time.time()
+    
+    # Broadcast progress via WebSocket with throttling
+    if ws_manager:
+        current_time = time.time()
+        should_broadcast = False
+        
+        with _progress_lock:
+            last_broadcast = _progress_last_broadcast.get(book_id, 0)
+            last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0)
+            time_elapsed = current_time - last_broadcast
+            
+            # Always broadcast at start (0%) or completion (>=99%)
+            if progress <= 1 or progress >= 99:
+                should_broadcast = True
+            # Broadcast if enough time has passed (convert interval from seconds)
+            elif time_elapsed >= config.DOWNLOAD_PROGRESS_UPDATE_INTERVAL:
+                should_broadcast = True
+            # Broadcast on significant progress jumps (>10%)
+            elif progress - last_progress >= 10:
+                should_broadcast = True
+            
+            if should_broadcast:
+                _progress_last_broadcast[book_id] = current_time
+                _progress_last_broadcast[f"{book_id}_progress"] = progress
+        
+        if should_broadcast:
+            ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
+
+def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
+    """Update download status with optional message for UI display."""
+    # Map string status to QueueStatus enum
+    status_map = {
+        'queued': QueueStatus.QUEUED,
+        'resolving': QueueStatus.RESOLVING,
+        'downloading': QueueStatus.DOWNLOADING,
+        'complete': QueueStatus.COMPLETE,
+        'available': QueueStatus.AVAILABLE,
+        'error': QueueStatus.ERROR,
+        'done': QueueStatus.DONE,
+        'cancelled': QueueStatus.CANCELLED,
+    }
+    
+    queue_status_enum = status_map.get(status.lower())
+    if queue_status_enum:
+        book_queue.update_status(book_id, queue_status_enum)
+
+        # Track activity for stall detection
+        with _progress_lock:
+            _last_activity[book_id] = time.time()
+
+        # Update status message if provided (empty string clears the message)
+        if message is not None:
+            book_queue.update_status_message(book_id, message)
+
+        # Broadcast status update via WebSocket
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+def cancel_download(book_id: str) -> bool:
+    """Cancel a download."""
+    result = book_queue.cancel_download(book_id)
+    
+    # Broadcast status update via WebSocket
+    if result and ws_manager and ws_manager.is_enabled():
+        ws_manager.broadcast_status_update(queue_status())
+    
+    return result
+
+def set_book_priority(book_id: str, priority: int) -> bool:
+    """Set priority for a queued book (lower = higher priority)."""
+    return book_queue.set_priority(book_id, priority)
+
+def reorder_queue(book_priorities: Dict[str, int]) -> bool:
+    """Bulk reorder queue by mapping book_id to new priority."""
+    return book_queue.reorder_queue(book_priorities)
+
+def get_queue_order() -> List[Dict[str, Any]]:
+    """Get current queue order for display."""
+    return book_queue.get_queue_order()
+
+def get_active_downloads() -> List[str]:
+    """Get list of currently active downloads."""
+    return book_queue.get_active_downloads()
+
+def clear_completed() -> int:
+    """Clear all completed downloads from tracking."""
+    return book_queue.clear_completed()
+
+def _cleanup_progress_tracking(task_id: str) -> None:
+    """Clean up progress tracking data for a completed/cancelled download."""
+    with _progress_lock:
+        _progress_last_broadcast.pop(task_id, None)
+        _progress_last_broadcast.pop(f"{task_id}_progress", None)
+        _last_activity.pop(task_id, None)
+
+
+def _process_single_download(task_id: str, cancel_flag: Event) -> None:
+    """Process a single download job."""
+    try:
+        # Status will be updated through callbacks during download process
+        # (resolving -> downloading -> complete)
+        download_path = _download_task(task_id, cancel_flag)
+
+        # Clean up progress tracking
+        _cleanup_progress_tracking(task_id)
+
+        if cancel_flag.is_set():
+            book_queue.update_status(task_id, QueueStatus.CANCELLED)
+            # Broadcast cancellation
+            if ws_manager:
+                ws_manager.broadcast_status_update(queue_status())
+            return
+
+        if download_path:
+            book_queue.update_download_path(task_id, download_path)
+            # Only update status if not already set (e.g., by archive extraction callback)
+            task = book_queue.get_task(task_id)
+            if not task or task.status != QueueStatus.COMPLETE:
+                book_queue.update_status(task_id, QueueStatus.COMPLETE)
+        else:
+            book_queue.update_status(task_id, QueueStatus.ERROR)
+
+        # Broadcast final status (completed or error)
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+    except Exception as e:
+        # Clean up progress tracking even on error
+        _cleanup_progress_tracking(task_id)
+
+        if not cancel_flag.is_set():
+            logger.error_trace(f"Error in download processing: {e}")
+            book_queue.update_status(task_id, QueueStatus.ERROR)
+            # Set error message if not already set by handler
+            task = book_queue.get_task(task_id)
+            if task and not task.status_message:
+                book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}: {str(e)}")
+        else:
+            logger.info(f"Download cancelled: {task_id}")
+            book_queue.update_status(task_id, QueueStatus.CANCELLED)
+
+        # Broadcast error/cancelled status
+        if ws_manager:
+            ws_manager.broadcast_status_update(queue_status())
+
+def concurrent_download_loop() -> None:
+    """Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
+    max_workers = config.MAX_CONCURRENT_DOWNLOADS
+    logger.info(f"Starting concurrent download loop with {max_workers} workers")
+
+    with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor:
+        active_futures: Dict[Future, str] = {}  # Track active download futures
+
+        while True:
+            # Clean up completed futures
+            completed_futures = [f for f in active_futures if f.done()]
+            for future in completed_futures:
+                task_id = active_futures.pop(future)
+                try:
+                    future.result()  # This will raise any exceptions from the worker
+                except Exception as e:
+                    logger.error_trace(f"Future exception for {task_id}: {e}")
+
+            # Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
+            current_time = time.time()
+            with _progress_lock:
+                for future, task_id in list(active_futures.items()):
+                    last_active = _last_activity.get(task_id, current_time)
+                    if current_time - last_active > STALL_TIMEOUT:
+                        logger.warning(f"Download stalled for {task_id}, cancelling")
+                        book_queue.cancel_download(task_id)
+                        book_queue.update_status_message(task_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)")
+
+            # Start new downloads if we have capacity
+            while len(active_futures) < max_workers:
+                next_download = book_queue.get_next()
+                if not next_download:
+                    break
+
+                # Stagger concurrent downloads to avoid rate limiting on shared download servers
+                # Only delay if other downloads are already active
+                if active_futures:
+                    stagger_delay = random.uniform(2, 5)
+                    logger.debug(f"Staggering download start by {stagger_delay:.1f}s")
+                    time.sleep(stagger_delay)
+
+                task_id, cancel_flag = next_download
+
+                # Submit download job to thread pool
+                future = executor.submit(_process_single_download, task_id, cancel_flag)
+                active_futures[future] = task_id
+
+            # Brief sleep to prevent busy waiting
+            time.sleep(config.MAIN_LOOP_SLEEP_TIME)
+
+# Download coordinator thread (started explicitly via start())
+_coordinator_thread: Optional[threading.Thread] = None
+_started = False
+
+
+def start() -> None:
+    """Start the download coordinator thread. Safe to call multiple times."""
+    global _coordinator_thread, _started
+
+    if _started:
+        logger.debug("Download coordinator already started")
+        return
+
+    _coordinator_thread = threading.Thread(
+        target=concurrent_download_loop,
+        daemon=True,
+        name="DownloadCoordinator"
+    )
+    _coordinator_thread.start()
+    _started = True
+
+    logger.info(f"Download coordinator started with {config.MAX_CONCURRENT_DOWNLOADS} concurrent workers")

shelfmark/download/outputs/__init__.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Callable, Optional
+
+from shelfmark.core.models import DownloadTask
+
+StatusCallback = Callable[[str, Optional[str]], None]
+OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback], Optional[str]]
+
+
+@dataclass(frozen=True)
+class OutputRegistration:
+    mode: str
+    supports_task: Callable[[DownloadTask], bool]
+    handler: OutputHandler
+    priority: int = 0
+
+
+_OUTPUT_REGISTRY: list[OutputRegistration] = []
+_OUTPUTS_LOADED = False
+
+
+def register_output(
+    mode: str,
+    supports_task: Callable[[DownloadTask], bool],
+    priority: int = 0,
+) -> Callable[[OutputHandler], OutputHandler]:
+    def decorator(handler: OutputHandler) -> OutputHandler:
+        _OUTPUT_REGISTRY.append(
+            OutputRegistration(
+                mode=mode,
+                supports_task=supports_task,
+                handler=handler,
+                priority=priority,
+            )
+        )
+        _OUTPUT_REGISTRY.sort(key=lambda entry: entry.priority, reverse=True)
+        return handler
+
+    return decorator
+
+
+def load_output_handlers() -> None:
+    global _OUTPUTS_LOADED
+    if _OUTPUTS_LOADED:
+        return
+
+    from . import booklore  # noqa: F401
+    from . import folder  # noqa: F401
+
+    _OUTPUTS_LOADED = True
+
+
+def resolve_output_handler(task: DownloadTask) -> Optional[OutputRegistration]:
+    load_output_handlers()
+    for entry in _OUTPUT_REGISTRY:
+        if entry.supports_task(task):
+            return entry
+    return None

shelfmark/download/outputs/booklore.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Any, Dict, List, Mapping, Optional
+
+import requests
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.outputs import register_output
+from shelfmark.download.staging import STAGE_MOVE, STAGE_NONE, build_staging_dir
+
+logger = setup_logger(__name__)
+
+BOOKLORE_OUTPUT_MODE = "booklore"
+BOOKLORE_SUPPORTED_EXTENSIONS = {".cb7", ".cbr", ".cbz", ".epub", ".fb2", ".pdf"}
+BOOKLORE_SUPPORTED_FORMATS_LABEL = ", ".join(
+    ext.lstrip(".").upper() for ext in sorted(BOOKLORE_SUPPORTED_EXTENSIONS)
+)
+
+
+class BookloreError(Exception):
+    """Raised when Booklore integration fails."""
+
+
+@dataclass(frozen=True)
+class BookloreConfig:
+    base_url: str
+    username: str
+    password: str
+    library_id: int
+    path_id: int
+    verify_tls: bool = True
+    refresh_after_upload: bool = False
+
+
+def _parse_int(value: Any, label: str) -> int:
+    if value is None or value == "":
+        raise BookloreError(f"{label} is required")
+    try:
+        return int(value)
+    except (TypeError, ValueError) as exc:
+        raise BookloreError(f"{label} must be a number") from exc
+
+
+def build_booklore_config(values: Mapping[str, Any]) -> BookloreConfig:
+    base_url = str(values.get("BOOKLORE_HOST", "")).strip()
+    username = str(values.get("BOOKLORE_USERNAME", "")).strip()
+    password = values.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        raise BookloreError("Booklore URL is required")
+    if not username:
+        raise BookloreError("Booklore username is required")
+    if not password:
+        raise BookloreError("Booklore password is required")
+
+    library_id = _parse_int(values.get("BOOKLORE_LIBRARY_ID"), "Booklore library ID")
+    path_id = _parse_int(values.get("BOOKLORE_PATH_ID"), "Booklore path ID")
+
+    return BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=library_id,
+        path_id=path_id,
+        verify_tls=True,
+        refresh_after_upload=True,  # Always refresh library after upload
+    )
+
+
+def booklore_login(booklore_config: BookloreConfig) -> str:
+    url = f"{booklore_config.base_url}/api/v1/auth/login"
+    payload = {"username": booklore_config.username, "password": booklore_config.password}
+
+    try:
+        response = requests.post(url, json=payload, timeout=30, verify=booklore_config.verify_tls)
+    except requests.exceptions.ConnectionError as exc:
+        raise BookloreError("Could not connect to Booklore") from exc
+    except requests.exceptions.Timeout as exc:
+        raise BookloreError("Booklore connection timed out") from exc
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore login failed: {exc}") from exc
+
+    if response.status_code in {401, 403}:
+        raise BookloreError("Booklore authentication failed")
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        raise BookloreError(f"Booklore login failed ({response.status_code})") from exc
+
+    try:
+        data = response.json()
+    except ValueError as exc:
+        raise BookloreError("Invalid Booklore login response") from exc
+
+    token = data.get("accessToken")
+    if not token:
+        raise BookloreError("Booklore did not return an access token")
+
+    return token
+
+
+def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list[dict[str, Any]]:
+    url = f"{booklore_config.base_url}/api/v1/libraries"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Failed to fetch Booklore libraries: {exc}") from exc
+
+    try:
+        return response.json()
+    except ValueError as exc:
+        raise BookloreError("Invalid Booklore libraries response") from exc
+
+
+def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None:
+    url = f"{booklore_config.base_url}/api/v1/files/upload"
+    headers = {"Authorization": f"Bearer {token}"}
+    params = {"libraryId": booklore_config.library_id, "pathId": booklore_config.path_id}
+
+    response = None
+
+    try:
+        with file_path.open("rb") as handle:
+            response = requests.post(
+                url,
+                headers=headers,
+                params=params,
+                files={"file": (file_path.name, handle)},
+                timeout=60,
+                verify=booklore_config.verify_tls,
+            )
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        message = response.text.strip() if response is not None else ""
+        if message:
+            message = f": {message[:200]}"
+        status_code = response.status_code if response is not None else "unknown"
+        raise BookloreError(f"Booklore upload failed ({status_code}){message}") from exc
+    except requests.exceptions.ConnectionError as exc:
+        raise BookloreError("Could not connect to Booklore") from exc
+    except requests.exceptions.Timeout as exc:
+        raise BookloreError("Booklore upload timed out") from exc
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore upload failed: {exc}") from exc
+
+
+def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None:
+    url = f"{booklore_config.base_url}/api/v1/libraries/{booklore_config.library_id}/refresh"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.put(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as exc:
+        raise BookloreError(f"Booklore refresh failed: {exc}") from exc
+
+
+def _supports_booklore(task: DownloadTask) -> bool:
+    if check_audiobook(task.content_type):
+        return False
+    return core_config.config.get("BOOKS_OUTPUT_MODE", "folder") == BOOKLORE_OUTPUT_MODE
+
+
+def _get_booklore_settings() -> Dict[str, Any]:
+    return {
+        "BOOKLORE_HOST": core_config.config.get("BOOKLORE_HOST", ""),
+        "BOOKLORE_USERNAME": core_config.config.get("BOOKLORE_USERNAME", ""),
+        "BOOKLORE_PASSWORD": core_config.config.get("BOOKLORE_PASSWORD", ""),
+        "BOOKLORE_LIBRARY_ID": core_config.config.get("BOOKLORE_LIBRARY_ID"),
+        "BOOKLORE_PATH_ID": core_config.config.get("BOOKLORE_PATH_ID"),
+    }
+
+
+def _booklore_format_error(rejected_files: List[Path]) -> str:
+    rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+    rejected_list = ", ".join(rejected_exts)
+    return (
+        f"Booklore does not support {rejected_list}. "
+        f"Supported formats: {BOOKLORE_SUPPORTED_FORMATS_LABEL}"
+    )
+
+
+def _post_process_booklore(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    from shelfmark.download.postprocess.pipeline import (
+        OutputPlan,
+        cleanup_output_staging,
+        is_managed_workspace_path,
+        prepare_output_files,
+    )
+
+    if cancel_flag.is_set():
+        logger.info("Task %s: cancelled before Booklore upload", task.task_id)
+        return None
+
+    try:
+        booklore_config = build_booklore_config(_get_booklore_settings())
+    except BookloreError as e:
+        logger.warning("Task %s: Booklore configuration error: %s", task.task_id, e)
+        status_callback("error", str(e))
+        return None
+
+    status_callback("resolving", "Preparing Booklore upload")
+
+    output_plan = OutputPlan(
+        mode=BOOKLORE_OUTPUT_MODE,
+        stage_action=STAGE_MOVE if is_managed_workspace_path(temp_file) else STAGE_NONE,
+        staging_dir=build_staging_dir("booklore", task.task_id),
+        allow_archive_extraction=True,
+    )
+
+    prepared = prepare_output_files(
+        temp_file,
+        task,
+        BOOKLORE_OUTPUT_MODE,
+        status_callback,
+        output_plan=output_plan,
+    )
+    if not prepared:
+        return None
+
+    logger.debug("Task %s: prepared %d file(s) for Booklore upload", task.task_id, len(prepared.files))
+
+    try:
+        unsupported_files = [
+            file_path
+            for file_path in prepared.files
+            if file_path.suffix.lower() not in BOOKLORE_SUPPORTED_EXTENSIONS
+        ]
+        if unsupported_files:
+            error_message = _booklore_format_error(unsupported_files)
+            logger.warning("Task %s: %s", task.task_id, error_message)
+            status_callback("error", error_message)
+            return None
+
+        token = booklore_login(booklore_config)
+        logger.info("Task %s: uploading %d file(s) to Booklore", task.task_id, len(prepared.files))
+
+        for index, file_path in enumerate(prepared.files, start=1):
+            if cancel_flag.is_set():
+                logger.info("Task %s: cancelled during Booklore upload", task.task_id)
+                return None
+            status_callback("resolving", f"Uploading to Booklore ({index}/{len(prepared.files)})")
+            booklore_upload_file(booklore_config, token, file_path)
+
+        if booklore_config.refresh_after_upload:
+            try:
+                booklore_refresh_library(booklore_config, token)
+            except BookloreError as e:
+                logger.warning("Task %s: Booklore refresh failed: %s", task.task_id, e)
+
+        logger.info("Task %s: uploaded %d file(s) to Booklore", task.task_id, len(prepared.files))
+
+        message = "Uploaded to Booklore"
+        if len(prepared.files) > 1:
+            message = f"Uploaded to Booklore ({len(prepared.files)} files)"
+        status_callback("complete", message)
+        return f"booklore://{task.task_id}"
+
+    except BookloreError as e:
+        logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e)
+        status_callback("error", str(e))
+        return None
+    except Exception as e:
+        logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e)
+        status_callback("error", f"Booklore upload failed: {e}")
+        return None
+    finally:
+        cleanup_output_staging(
+            prepared.output_plan,
+            prepared.working_path,
+            task,
+            prepared.cleanup_paths,
+        )
+
+
+@register_output(BOOKLORE_OUTPUT_MODE, supports_task=_supports_booklore, priority=10)
+def process_booklore_output(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    return _post_process_booklore(temp_file, task, cancel_flag, status_callback)

shelfmark/download/outputs/folder.py

@@ -0,0 +1,310 @@
+from __future__ import annotations
+
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event
+from typing import Any, Optional, List
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.archive import is_archive
+from shelfmark.download.outputs import register_output
+from shelfmark.download.staging import StageAction, STAGE_NONE
+
+logger = setup_logger(__name__)
+
+FOLDER_OUTPUT_MODE = "folder"
+
+
+def _resolve_custom_script_target(target_path: Path, destination: Path, path_mode: str) -> Path:
+    mode = (path_mode or "absolute").strip().lower()
+    if mode != "relative":
+        return target_path
+
+    try:
+        return target_path.relative_to(destination)
+    except ValueError:
+        if target_path.is_absolute():
+            return Path(target_path.name)
+        return target_path
+
+
+@dataclass(frozen=True)
+class _ProcessingPlan:
+    destination: Path
+    organization_mode: str
+    use_hardlink: bool
+    allow_archive_extraction: bool
+    stage_action: StageAction
+    staging_dir: Path
+    hardlink_source: Optional[Path]
+    output_mode: str = FOLDER_OUTPUT_MODE
+
+
+def _supports_folder_output(task: DownloadTask) -> bool:
+    if check_audiobook(task.content_type):
+        return True
+    return core_config.config.get("BOOKS_OUTPUT_MODE", FOLDER_OUTPUT_MODE) == FOLDER_OUTPUT_MODE
+
+
+def _build_processing_plan(
+    temp_file: Path,
+    task: DownloadTask,
+    status_callback,
+) -> Optional[_ProcessingPlan]:
+    from shelfmark.download.postprocess.pipeline import (
+        build_output_plan,
+        get_final_destination,
+        validate_destination,
+    )
+    from shelfmark.download.postprocess.policy import get_file_organization
+
+    is_audiobook = check_audiobook(task.content_type)
+    organization_mode = get_file_organization(is_audiobook)
+    destination = get_final_destination(task)
+
+    if not validate_destination(destination, status_callback):
+        return None
+
+    output_plan = build_output_plan(
+        temp_file,
+        task,
+        output_mode=FOLDER_OUTPUT_MODE,
+        destination=destination,
+        status_callback=status_callback,
+    )
+    if not output_plan.transfer_plan:
+        return None
+
+    transfer_plan = output_plan.transfer_plan
+    hardlink_source = transfer_plan.source_path if transfer_plan.use_hardlink else None
+
+    return _ProcessingPlan(
+        destination=destination,
+        organization_mode=organization_mode,
+        use_hardlink=transfer_plan.use_hardlink,
+        allow_archive_extraction=transfer_plan.allow_archive_extraction,
+        stage_action=output_plan.stage_action,
+        staging_dir=output_plan.staging_dir,
+        hardlink_source=hardlink_source,
+    )
+
+
+@register_output(FOLDER_OUTPUT_MODE, supports_task=_supports_folder_output, priority=0)
+def process_folder_output(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    """Post-process download to the configured folder destination."""
+    from shelfmark.download.postprocess.pipeline import (
+        cleanup_output_staging,
+        is_torrent_source,
+        log_plan_steps,
+        prepare_output_files,
+        record_step,
+        safe_cleanup_path,
+        transfer_book_files,
+    )
+
+    plan = _build_processing_plan(temp_file, task, status_callback)
+    if not plan:
+        return None
+
+    logger.debug(
+        "Processing plan for task %s: mode=%s destination=%s hardlink=%s stage_action=%s extract_archives=%s",
+        task.task_id,
+        plan.organization_mode,
+        plan.destination,
+        plan.use_hardlink,
+        plan.stage_action,
+        plan.allow_archive_extraction,
+    )
+
+    prepared = prepare_output_files(
+        temp_file,
+        task,
+        output_mode=plan.output_mode,
+        status_callback=status_callback,
+        destination=plan.destination,
+    )
+    if not prepared:
+        return None
+
+    steps: List[Any] = []
+    if prepared.output_plan.stage_action != STAGE_NONE:
+        step_name = f"stage_{prepared.output_plan.stage_action}"
+        record_step(steps, step_name, source=str(temp_file), dest=str(prepared.output_plan.staging_dir))
+
+    def run_custom_script(script_path: str, target_path: Path, phase: str) -> bool:
+        path_mode = core_config.config.get("CUSTOM_SCRIPT_PATH_MODE", "absolute")
+        script_target = _resolve_custom_script_target(target_path, plan.destination, path_mode)
+        env = {
+            **os.environ,
+            "SHELFMARK_CUSTOM_SCRIPT_TARGET": str(target_path),
+            "SHELFMARK_CUSTOM_SCRIPT_RELATIVE": str(_resolve_custom_script_target(target_path, plan.destination, "relative")),
+            "SHELFMARK_CUSTOM_SCRIPT_DESTINATION": str(plan.destination),
+            "SHELFMARK_CUSTOM_SCRIPT_MODE": str(path_mode),
+            "SHELFMARK_CUSTOM_SCRIPT_PHASE": phase,
+        }
+        record_step(
+            steps,
+            "custom_script",
+            script=str(script_path),
+            target=str(script_target),
+            target_abs=str(target_path),
+            mode=str(path_mode),
+            phase=phase,
+        )
+        log_plan_steps(task.task_id, steps)
+        logger.info(
+            "Task %s: running custom script %s on %s (%s)",
+            task.task_id,
+            script_path,
+            script_target,
+            phase,
+        )
+        try:
+            result = subprocess.run(
+                [script_path, str(script_target)],
+                check=True,
+                timeout=300,  # 5 minute timeout
+                capture_output=True,
+                text=True,
+                env=env,
+            )
+            if result.stdout:
+                logger.debug("Task %s: custom script stdout: %s", task.task_id, result.stdout.strip())
+            return True
+        except FileNotFoundError:
+            logger.error("Task %s: custom script not found: %s", task.task_id, script_path)
+            status_callback("error", f"Custom script not found: {script_path}")
+            return False
+        except PermissionError:
+            logger.error("Task %s: custom script not executable: %s", task.task_id, script_path)
+            status_callback("error", f"Custom script not executable: {script_path}")
+            return False
+        except subprocess.TimeoutExpired:
+            logger.error("Task %s: custom script timed out after 300s: %s", task.task_id, script_path)
+            status_callback("error", "Custom script timed out")
+            return False
+        except subprocess.CalledProcessError as e:
+            stderr = e.stderr.strip() if e.stderr else "No error output"
+            logger.error(
+                "Task %s: custom script failed (exit code %s): %s",
+                task.task_id,
+                e.returncode,
+                stderr,
+            )
+            status_callback("error", f"Custom script failed: {stderr[:100]}")
+            return False
+
+    # Custom script is run post-transfer (see below).
+
+    # If we staged a copy into TMP_DIR (e.g. for custom script), transfer from the staged
+    # path and disable hardlinking for this transfer.
+    use_hardlink = plan.use_hardlink and prepared.output_plan.stage_action == STAGE_NONE
+    source_path = plan.hardlink_source if use_hardlink and plan.hardlink_source else prepared.working_path
+    is_torrent = is_torrent_source(source_path, task)
+
+    usenet_action = core_config.config.get("PROWLARR_USENET_ACTION", "move")
+    is_usenet = task.source == "prowlarr" and not task.original_download_path
+
+    # For external usenet downloads, always copy from the client path.
+    # "Move" is implemented as a client-side cleanup after import.
+    preserve_source = is_usenet
+
+    copy_for_label = is_torrent or preserve_source or prepared.output_plan.stage_action != STAGE_NONE
+
+    if cancel_flag.is_set():
+        logger.info("Task %s: cancelled before final transfer", task.task_id)
+        cleanup_output_staging(
+            prepared.output_plan,
+            prepared.working_path,
+            task,
+            prepared.cleanup_paths,
+        )
+        return None
+
+    if use_hardlink:
+        op_label = "Hardlinking"
+    elif is_usenet and usenet_action == "move" and prepared.output_plan.stage_action == STAGE_NONE:
+        # Presented as a move, but implemented as copy + client cleanup.
+        op_label = "Moving"
+    elif copy_for_label:
+        op_label = "Copying"
+    else:
+        op_label = "Moving"
+
+    status_callback("resolving", f"{op_label} file")
+    record_step(
+        steps,
+        "transfer",
+        op=op_label.lower(),
+        source=str(source_path),
+        dest=str(plan.destination),
+        hardlink=use_hardlink,
+        torrent=copy_for_label,
+    )
+    if prepared.output_plan.stage_action != STAGE_NONE:
+        record_step(steps, "cleanup_staging", path=str(prepared.working_path))
+    log_plan_steps(task.task_id, steps)
+
+    final_paths, error = transfer_book_files(
+        prepared.files,
+        destination=plan.destination,
+        task=task,
+        use_hardlink=use_hardlink,
+        is_torrent=is_torrent,
+        preserve_source=preserve_source,
+        organization_mode=plan.organization_mode,
+    )
+
+    if error:
+        logger.warning("Task %s: transfer failed: %s", task.task_id, error)
+        status_callback("error", error)
+        return None
+
+    logger.info(
+        "Task %s: transferred %d file(s) to %s (%s)",
+        task.task_id,
+        len(final_paths),
+        plan.destination,
+        op_label.lower(),
+    )
+
+    # Run custom script once per successful task, after transfer.
+    if core_config.config.CUSTOM_SCRIPT:
+        if len(final_paths) == 1:
+            target_path = final_paths[0]
+        else:
+            try:
+                target_path = Path(os.path.commonpath([str(p.parent) for p in final_paths]))
+            except ValueError:
+                target_path = plan.destination
+
+        if not run_custom_script(core_config.config.CUSTOM_SCRIPT, target_path, phase="post_transfer"):
+            cleanup_output_staging(
+                prepared.output_plan,
+                prepared.working_path,
+                task,
+                prepared.cleanup_paths,
+            )
+            return None
+
+    cleanup_output_staging(
+        prepared.output_plan,
+        prepared.working_path,
+        task,
+        prepared.cleanup_paths,
+    )
+
+    message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
+    status_callback("complete", message)
+
+    return str(final_paths[0])

shelfmark/download/permissions_debug.py

@@ -0,0 +1,125 @@
+"""Permission/ownership diagnostics for filesystem operations.
+
+This module centralizes best-effort debug logging used by download post-processing
+and atomic filesystem operations.
+
+It is intentionally defensive: failures collecting context should never mask the
+original error.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def _format_uid(uid: int) -> str:
+    try:
+        import pwd
+
+        return pwd.getpwuid(uid).pw_name
+    except Exception:
+        return str(uid)
+
+
+def _format_gid(gid: int) -> str:
+    try:
+        import grp
+
+        return grp.getgrgid(gid).gr_name
+    except Exception:
+        return str(gid)
+
+
+def log_path_permission_context(label: str, path: Path) -> None:
+    """Log useful permission/ownership context for a path.
+
+    Only call this from failure paths.
+    """
+
+    try:
+        euid = os.geteuid() if hasattr(os, "geteuid") else None
+        egid = os.getegid() if hasattr(os, "getegid") else None
+        groups = os.getgroups() if hasattr(os, "getgroups") else []
+
+        if euid is not None and egid is not None:
+            logger.debug(
+                "Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s",
+                label,
+                _format_uid(euid),
+                euid,
+                _format_gid(egid),
+                egid,
+                [f"{_format_gid(g)}({g})" for g in groups],
+            )
+
+        for probe in [path, path.parent]:
+            try:
+                resolved = probe.resolve()
+            except Exception:
+                resolved = probe
+
+            try:
+                st = probe.stat()
+                logger.debug(
+                    "Path permissions (%s): path=%s resolved=%s mode=%s owner=%s(%d) group=%s(%d) dir=%s symlink=%s",
+                    label,
+                    probe,
+                    resolved,
+                    oct(st.st_mode & 0o777),
+                    _format_uid(st.st_uid),
+                    st.st_uid,
+                    _format_gid(st.st_gid),
+                    st.st_gid,
+                    probe.is_dir(),
+                    probe.is_symlink(),
+                )
+            except Exception as stat_error:
+                logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
+    except Exception as context_error:
+        logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
+
+
+def log_transfer_permission_context(label: str, source: Path, dest: Path, error: Exception) -> None:
+    """Log useful permission/ownership context when a file transfer fails."""
+
+    try:
+        euid = os.geteuid() if hasattr(os, "geteuid") else None
+        egid = os.getegid() if hasattr(os, "getegid") else None
+        groups = os.getgroups() if hasattr(os, "getgroups") else []
+
+        if euid is not None and egid is not None:
+            logger.debug(
+                "Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s error=%s",
+                label,
+                _format_uid(euid),
+                euid,
+                _format_gid(egid),
+                egid,
+                [f"{_format_gid(g)}({g})" for g in groups],
+                error,
+            )
+
+        for probe in [source, dest, dest.parent]:
+            try:
+                st = probe.stat()
+                logger.debug(
+                    "Path permissions (%s): path=%s mode=%s owner=%s(%d) group=%s(%d) exists=%s dir=%s",
+                    label,
+                    probe,
+                    oct(st.st_mode & 0o777),
+                    _format_uid(st.st_uid),
+                    st.st_uid,
+                    _format_gid(st.st_gid),
+                    st.st_gid,
+                    probe.exists(),
+                    probe.is_dir(),
+                )
+            except Exception as stat_error:
+                logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
+    except Exception as context_error:
+        logger.debug("Permission context (%s): failed to collect: %s", label, context_error)

shelfmark/download/postprocess/__init__.py

@@ -0,0 +1,11 @@
+"""Post-download processing pipeline.
+
+This package contains the post-download processing pipeline (staging, scanning,
+archive extraction, transfers, and safe cleanup) and the router that selects an
+output handler.
+
+Output handlers live in `shelfmark.download.outputs` and should depend on
+`pipeline` (not `router`) to avoid circular imports.
+"""
+
+from .router import post_process_download

shelfmark/download/postprocess/destination.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import (
+    get_aa_content_type_dir,
+    get_destination,
+    is_audiobook as check_audiobook,
+)
+from shelfmark.download.permissions_debug import log_path_permission_context
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def validate_destination(destination: Path, status_callback) -> bool:
+    """Validate destination path is absolute, exists, and writable."""
+
+    if not destination.is_absolute():
+        logger.warning(f"Destination must be absolute: {destination}")
+        status_callback("error", f"Destination must be absolute: {destination}")
+        return False
+
+    if destination.exists() and not destination.is_dir():
+        logger.warning(f"Destination is not a directory: {destination}")
+        status_callback("error", f"Destination is not a directory: {destination}")
+        return False
+
+    if not destination.exists():
+        try:
+            destination.mkdir(parents=True, exist_ok=True)
+        except (OSError, PermissionError) as exc:
+            log_path_permission_context("destination_create", destination)
+            logger.warning(f"Cannot create destination: {destination} ({exc})")
+            status_callback("error", f"Cannot create destination: {destination} ({exc})")
+            return False
+
+    test_path = destination / f".shelfmark_write_test_{uuid.uuid4().hex}.tmp"
+
+    try:
+        test_content = (
+            f"This file was created to verify if '{destination}' is writable. "
+            "It should've been automatically deleted. Feel free to delete it.\n"
+        )
+        test_path.write_text(test_content)
+        test_path.unlink(missing_ok=True)
+    except Exception as exc:
+        logger.debug("Destination write probe path: %s", test_path)
+        log_path_permission_context("destination_write_probe", destination)
+        logger.warning(f"Destination not writable: {destination} ({exc})")
+        status_callback("error", f"Destination not writable: {destination} ({exc})")
+        return False
+
+    return True
+
+
+def get_final_destination(task: DownloadTask) -> Path:
+    """Get final destination directory, with content-type routing support."""
+
+    is_audiobook = check_audiobook(task.content_type)
+
+    if task.source == "direct_download" and not is_audiobook:
+        override = get_aa_content_type_dir(task.content_type)
+        if override:
+            return override
+
+    return get_destination(is_audiobook)

shelfmark/download/postprocess/pipeline.py

@@ -0,0 +1,76 @@
+"""Post-download processing pipeline.
+
+This module is the public API surface for post-download processing.
+
+Implementation lives in submodules in this package:
+
+- `types`: dataclasses used across the pipeline
+- `workspace`: managed workspace + cleanup rules
+- `scan`: directory scanning + archive extraction
+- `transfer`: hardlink/copy/move + naming/organization
+- `prepare`: staging plan + prepared file selection
+- `steps`: lightweight plan logging helpers
+
+Keeping this file as a facade avoids churn in call sites while letting the
+implementation stay modular.
+"""
+
+from __future__ import annotations
+
+from .destination import get_final_destination, validate_destination
+from .prepare import build_output_plan, prepare_output_files
+from .scan import (
+    collect_directory_files,
+    collect_staged_files,
+    extract_archive_files,
+    get_supported_formats,
+    scan_directory_tree,
+)
+from .steps import log_plan_steps, record_step
+from .transfer import (
+    build_metadata_dict,
+    is_torrent_source,
+    process_directory,
+    resolve_hardlink_source,
+    should_hardlink,
+    transfer_book_files,
+    transfer_directory_to_library,
+    transfer_file_to_library,
+)
+from .types import OutputPlan, PlanStep, PreparedFiles, TransferPlan
+from .workspace import (
+    cleanup_output_staging,
+    is_managed_workspace_path,
+    is_within_tmp_dir,
+    safe_cleanup_path,
+)
+
+__all__ = [
+    "OutputPlan",
+    "PlanStep",
+    "PreparedFiles",
+    "TransferPlan",
+    "build_metadata_dict",
+    "build_output_plan",
+    "cleanup_output_staging",
+    "collect_directory_files",
+    "collect_staged_files",
+    "extract_archive_files",
+    "get_final_destination",
+    "get_supported_formats",
+    "is_managed_workspace_path",
+    "is_torrent_source",
+    "is_within_tmp_dir",
+    "log_plan_steps",
+    "prepare_output_files",
+    "process_directory",
+    "record_step",
+    "resolve_hardlink_source",
+    "safe_cleanup_path",
+    "scan_directory_tree",
+    "should_hardlink",
+    "transfer_book_files",
+    "transfer_directory_to_library",
+    "transfer_file_to_library",
+    "validate_destination",
+]

shelfmark/download/postprocess/policy.py

@@ -0,0 +1,100 @@
+"""Post-download processing policy.
+
+This module holds configuration-driven *policy* decisions that are shared across
+post-download processing components, but are not specific to archive extraction.
+
+Examples:
+- Which file formats are enabled
+- How files should be organized (none/rename/organize)
+- Which naming templates to use
+
+Implementation note:
+Keep this module free of dependencies on archive extraction mechanics to avoid
+circular imports (`archive` is used by the pipeline).
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+import shelfmark.core.config as core_config
+
+
+def get_supported_formats() -> List[str]:
+    """Get current supported formats from config singleton."""
+
+    formats = core_config.config.get(
+        "SUPPORTED_FORMATS",
+        ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
+    )
+
+    # Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
+    if isinstance(formats, str):
+        return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
+
+    return [fmt.lower() for fmt in formats]
+
+
+def get_supported_audiobook_formats() -> List[str]:
+    """Get current supported audiobook formats from config singleton."""
+
+    formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
+
+    # Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
+    if isinstance(formats, str):
+        return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
+
+    return [fmt.lower() for fmt in formats]
+
+
+def get_file_organization(is_audiobook: bool) -> str:
+    """Get the file organization mode for the content type."""
+
+    key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
+    mode = core_config.config.get(key, "rename")
+
+    # Handle legacy settings migration
+    if mode not in ("none", "rename", "organize"):
+        legacy_key = "PROCESSING_MODE_AUDIOBOOK" if is_audiobook else "PROCESSING_MODE"
+        legacy_mode = core_config.config.get(legacy_key, "ingest")
+        if legacy_mode == "library":
+            return "organize"
+        if core_config.config.get("USE_BOOK_TITLE", True):
+            return "rename"
+        return "none"
+
+    return mode
+
+
+def get_template(is_audiobook: bool, organization_mode: str) -> str:
+    """Get the template for the content type and organization mode."""
+
+    # Determine the correct key based on content type and organization mode
+    if is_audiobook:
+        if organization_mode == "organize":
+            key = "TEMPLATE_AUDIOBOOK_ORGANIZE"
+        else:
+            key = "TEMPLATE_AUDIOBOOK_RENAME"
+    else:
+        if organization_mode == "organize":
+            key = "TEMPLATE_ORGANIZE"
+        else:
+            key = "TEMPLATE_RENAME"
+
+    template = core_config.config.get(key, "")
+
+    # Fallback to legacy keys if new keys are empty
+    if not template:
+        legacy_key = "TEMPLATE_AUDIOBOOK" if is_audiobook else "TEMPLATE"
+        template = core_config.config.get(legacy_key, "")
+
+    if not template:
+        legacy_key = "LIBRARY_TEMPLATE_AUDIOBOOK" if is_audiobook else "LIBRARY_TEMPLATE"
+        template = core_config.config.get(legacy_key, "")
+
+    if not template:
+        if organization_mode == "organize":
+            return "{Author}/{Title} ({Year})"
+        return "{Author} - {Title} ({Year})"
+
+    return template

shelfmark/download/postprocess/prepare.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.download.archive import is_archive
+from shelfmark.download.staging import STAGE_COPY, STAGE_NONE, get_staging_dir, stage_path
+
+from .scan import collect_staged_files
+from .transfer import resolve_hardlink_source
+from .types import OutputPlan, PreparedFiles
+from .workspace import cleanup_output_staging, is_managed_workspace_path
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def build_output_plan(
+    temp_file: Path,
+    task: DownloadTask,
+    output_mode: str,
+    destination: Optional[Path] = None,
+    status_callback=None,
+) -> OutputPlan:
+    """Build an output plan that describes staging behavior for file-based outputs."""
+
+    transfer_plan = resolve_hardlink_source(temp_file, task, destination, status_callback)
+    runs_custom_script = bool(core_config.config.CUSTOM_SCRIPT) and temp_file.is_file() and not is_archive(temp_file)
+
+    stage_action = STAGE_COPY if runs_custom_script and not is_managed_workspace_path(temp_file) else STAGE_NONE
+    staging_dir = get_staging_dir()
+
+    return OutputPlan(
+        mode=output_mode,
+        stage_action=stage_action,
+        staging_dir=staging_dir,
+        allow_archive_extraction=transfer_plan.allow_archive_extraction,
+        transfer_plan=transfer_plan,
+    )
+
+
+def prepare_output_files(
+    temp_file: Path,
+    task: DownloadTask,
+    output_mode: str,
+    status_callback,
+    destination: Optional[Path] = None,
+    output_plan: Optional[OutputPlan] = None,
+) -> Optional[PreparedFiles]:
+    if output_plan is None:
+        output_plan = build_output_plan(
+            temp_file,
+            task,
+            output_mode=output_mode,
+            destination=destination,
+            status_callback=status_callback,
+        )
+
+    working_path = temp_file
+    if output_plan.stage_action != STAGE_NONE:
+        step_label = "Staging torrent files" if output_plan.stage_action == STAGE_COPY else "Staging files"
+        status_callback("resolving", step_label)
+        working_path = stage_path(working_path, output_plan.staging_dir, output_plan.stage_action)
+
+    can_delete_source_archives = output_plan.stage_action != STAGE_NONE or is_managed_workspace_path(working_path)
+
+    files, rejected_files, cleanup_paths, error = collect_staged_files(
+        working_path=working_path,
+        task=task,
+        allow_archive_extraction=output_plan.allow_archive_extraction,
+        status_callback=status_callback,
+        cleanup_archives=can_delete_source_archives,
+    )
+
+    if error:
+        status_callback("error", error)
+        cleanup_output_staging(output_plan, working_path, task, cleanup_paths)
+        return None
+
+    if output_plan.stage_action == STAGE_NONE and is_managed_workspace_path(working_path):
+        cleanup_paths = [*cleanup_paths, working_path]
+
+    return PreparedFiles(
+        output_plan=output_plan,
+        working_path=working_path,
+        files=files,
+        rejected_files=rejected_files,
+        cleanup_paths=cleanup_paths,
+    )

shelfmark/download/postprocess/router.py

@@ -0,0 +1,52 @@
+"""Output routing for post-download processing.
+
+This module selects the appropriate output handler and invokes it.
+
+Keeping this separate from `pipeline.py` avoids circular imports:
+
+- output handlers depend on `pipeline`
+- router depends on the output registry
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from threading import Event
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask, SearchMode
+from shelfmark.download.outputs import resolve_output_handler
+
+logger = setup_logger(__name__)
+
+
+def post_process_download(
+    temp_file: Path,
+    task: DownloadTask,
+    cancel_flag: Event,
+    status_callback,
+) -> Optional[str]:
+    """Post-process download using the selected output handler."""
+
+    if task.search_mode is None:
+        logger.warning(
+            "Task %s: missing search_mode; defaulting to Direct mode behavior",
+            task.task_id,
+        )
+    elif task.search_mode not in (SearchMode.DIRECT, SearchMode.UNIVERSAL):
+        logger.warning(
+            "Task %s: invalid search_mode=%s; defaulting to Direct mode behavior",
+            task.task_id,
+            task.search_mode,
+        )
+
+    output_handler = resolve_output_handler(task)
+    if output_handler:
+        logger.info("Task %s: using output mode %s", task.task_id, output_handler.mode)
+        return output_handler.handler(temp_file, task, cancel_flag, status_callback)
+
+    from shelfmark.download.outputs.folder import process_folder_output
+
+    logger.info("Task %s: using output mode folder", task.task_id)
+    return process_folder_output(temp_file, task, cancel_flag, status_callback)

shelfmark/download/postprocess/scan.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
+from shelfmark.download.permissions_debug import log_path_permission_context
+from shelfmark.download.postprocess.policy import (
+    get_supported_audiobook_formats,
+    get_supported_formats as get_book_formats,
+)
+from shelfmark.download.staging import build_staging_dir
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def get_supported_formats(content_type: Optional[str] = None) -> List[str]:
+    if check_audiobook(content_type):
+        return get_supported_audiobook_formats()
+    return get_book_formats()
+
+
+def _format_not_supported_error(rejected_files: List[Path], task: DownloadTask) -> str:
+    content_type = task.content_type
+    file_type_label = "audiobook" if check_audiobook(content_type) else "book"
+    rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+    rejected_list = ", ".join(rejected_exts)
+    supported_formats = get_supported_formats(content_type)
+
+    logger.warning(
+        "Task %s: found %d %s(s) but none match supported formats. Rejected formats: %s. Supported: %s",
+        task.task_id,
+        len(rejected_files),
+        file_type_label,
+        rejected_list,
+        ", ".join(sorted(supported_formats)),
+    )
+
+    return (
+        f"Found {len(rejected_files)} {file_type_label}(s) but format not supported ({rejected_list}). "
+        "Enable in Settings > Formats."
+    )
+
+
+def extract_archive_files(
+    archive_path: Path,
+    output_dir: Path,
+    task: DownloadTask,
+    cleanup_archive: bool,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    content_type = task.content_type
+
+    try:
+        extracted_files, warnings, rejected_files = extract_archive(archive_path, output_dir, content_type)
+    except ArchiveExtractionError as exc:
+        logger.warning(
+            "Task %s: archive extraction failed for %s: %s",
+            task.task_id,
+            archive_path.name,
+            exc,
+        )
+        return [], [], [], str(exc)
+
+    if warnings:
+        logger.debug(
+            "Task %s: archive warnings for %s: %s",
+            task.task_id,
+            archive_path.name,
+            "; ".join(warnings),
+        )
+
+    if cleanup_archive:
+        archive_path.unlink(missing_ok=True)
+
+    cleanup_paths = [output_dir]
+
+    if not extracted_files:
+        if rejected_files:
+            return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
+        file_type_label = "audiobook" if check_audiobook(content_type) else "book"
+        return [], rejected_files, cleanup_paths, f"No {file_type_label} files found in archive"
+
+    logger.debug(
+        "Task %s: extracted %d file(s) from archive %s",
+        task.task_id,
+        len(extracted_files),
+        archive_path.name,
+    )
+
+    return extracted_files, rejected_files, cleanup_paths, None
+
+
+def scan_directory_tree(
+    directory: Path,
+    content_type: Optional[str],
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    """Scan a directory tree for book files, trackable-but-unsupported files, and archives."""
+
+    try:
+        with os.scandir(directory) as it:
+            next(it, None)
+    except PermissionError as exc:
+        log_path_permission_context("scan_directory", directory)
+        logger.warning(f"Permission denied scanning directory: {directory} ({exc})")
+        return [], [], [], f"Permission denied accessing download folder: {directory}"
+    except (FileNotFoundError, NotADirectoryError, OSError) as exc:
+        logger.warning(f"Cannot access download folder: {directory} ({exc})")
+        return [], [], [], f"Cannot access download folder: {directory} ({exc})"
+
+    book_files: List[Path] = []
+    rejected_files: List[Path] = []
+    archive_files: List[Path] = []
+
+    supported_formats = get_supported_formats(content_type)
+    supported_exts = {f".{fmt}" for fmt in supported_formats}
+
+    is_audiobook = check_audiobook(content_type)
+    if is_audiobook:
+        trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
+    else:
+        trackable_exts = {
+            '.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
+            '.doc', '.docx', '.rtf', '.txt',
+        }
+
+    logged_walk_permission_context = False
+
+    def onerror(error: OSError) -> None:
+        nonlocal logged_walk_permission_context
+
+        if isinstance(error, PermissionError):
+            if not logged_walk_permission_context:
+                try:
+                    error_path = Path(getattr(error, "filename", "") or str(directory))
+                except Exception:
+                    error_path = directory
+
+                log_path_permission_context("scan_directory_walk", error_path)
+                logged_walk_permission_context = True
+
+            logger.debug(f"Skipping inaccessible path during scan: {error}")
+        else:
+            logger.debug(f"Error scanning directory tree: {error}")
+
+    for root, _, files in os.walk(directory, onerror=onerror):
+        for filename in files:
+            file_path = Path(root) / filename
+            suffix = file_path.suffix.lower()
+
+            if suffix in supported_exts:
+                book_files.append(file_path)
+            elif suffix in trackable_exts:
+                rejected_files.append(file_path)
+
+            if is_archive(file_path):
+                archive_files.append(file_path)
+
+    return book_files, rejected_files, archive_files, None
+
+
+def collect_directory_files(
+    directory: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool,
+    status_callback=None,
+    cleanup_archives: bool = False,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    content_type = task.content_type
+    book_files, rejected_files, archive_files, scan_error = scan_directory_tree(directory, content_type)
+    if scan_error:
+        return [], [], [], scan_error
+
+    if book_files:
+        if archive_files:
+            logger.debug(
+                "Task %s: ignoring %d archive(s) - already have %d book file(s)",
+                task.task_id,
+                len(archive_files),
+                len(book_files),
+            )
+        if rejected_files:
+            rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
+            logger.debug(
+                "Task %s: also found %d file(s) with unsupported formats: %s",
+                task.task_id,
+                len(rejected_files),
+                ", ".join(rejected_exts),
+            )
+        return book_files, rejected_files, [], None
+
+    if archive_files:
+        if not allow_archive_extraction:
+            logger.warning(
+                "Task %s: archive extraction disabled (torrent hardlinking enabled) for %s",
+                task.task_id,
+                directory,
+            )
+            return [], rejected_files, [], "Archive extraction is disabled when torrent hardlinking is enabled"
+
+        if status_callback:
+            status_callback("resolving", "Extracting archives")
+
+        logger.info("Task %s: extracting %d archive(s)", task.task_id, len(archive_files))
+
+        all_files: List[Path] = []
+        all_errors: List[str] = []
+        cleanup_paths: List[Path] = []
+
+        for archive in archive_files:
+            extract_dir = build_staging_dir("extract", task.task_id)
+            extracted_files, archive_rejected, archive_cleanup, error = extract_archive_files(
+                archive_path=archive,
+                output_dir=extract_dir,
+                task=task,
+                cleanup_archive=cleanup_archives,
+            )
+
+            if error:
+                all_errors.append(f"{archive.name}: {error}")
+            if archive_rejected:
+                rejected_files.extend(archive_rejected)
+            if extracted_files:
+                all_files.extend(extracted_files)
+            if archive_cleanup:
+                cleanup_paths.extend(archive_cleanup)
+
+        if all_files:
+            logger.info(
+                "Task %s: extracted %d file(s) from %d archive(s)",
+                task.task_id,
+                len(all_files),
+                len(archive_files),
+            )
+            return all_files, rejected_files, cleanup_paths, None
+
+        if all_errors:
+            return [], rejected_files, cleanup_paths, "; ".join(all_errors)
+
+        if rejected_files:
+            return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
+
+        return [], rejected_files, cleanup_paths, "No book files found in archives"
+
+    if rejected_files:
+        return [], rejected_files, [], _format_not_supported_error(rejected_files, task)
+
+    return [], rejected_files, [], "No book files found in download"
+
+
+def collect_staged_files(
+    working_path: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool,
+    status_callback,
+    cleanup_archives: bool,
+) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
+    if working_path.is_dir():
+        if status_callback:
+            status_callback("resolving", "Processing download folder")
+        return collect_directory_files(
+            working_path,
+            task,
+            allow_archive_extraction=allow_archive_extraction,
+            status_callback=status_callback,
+            cleanup_archives=cleanup_archives,
+        )
+
+    if is_archive(working_path) and allow_archive_extraction:
+        if status_callback:
+            status_callback("resolving", "Extracting archive")
+
+        logger.info("Task %s: extracting archive %s", task.task_id, working_path.name)
+
+        extract_dir = build_staging_dir("extract", task.task_id)
+        extracted_files, rejected_files, cleanup_paths, error = extract_archive_files(
+            archive_path=working_path,
+            output_dir=extract_dir,
+            task=task,
+            cleanup_archive=cleanup_archives,
+        )
+
+        if extracted_files:
+            logger.info(
+                "Task %s: extracted %d file(s) from archive %s",
+                task.task_id,
+                len(extracted_files),
+                working_path.name,
+            )
+
+        return extracted_files, rejected_files, cleanup_paths, error
+
+    # Single-file download result (non-archive).
+    # Ensure we respect the user's supported format settings.
+    suffix = working_path.suffix.lower()
+    supported_formats = get_supported_formats(task.content_type)
+    supported_exts = {f".{fmt}" for fmt in supported_formats}
+
+    is_audiobook = check_audiobook(task.content_type)
+    if is_audiobook:
+        trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
+    else:
+        trackable_exts = {
+            '.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
+            '.doc', '.docx', '.rtf', '.txt',
+        }
+
+    if suffix in supported_exts:
+        return [working_path], [], [], None
+
+    if suffix in trackable_exts:
+        return [], [working_path], [], _format_not_supported_error([working_path], task)
+
+    file_type_label = "audiobook" if is_audiobook else "book"
+    return [], [], [], f"Unsupported {file_type_label} file type: {suffix or working_path.name}"

shelfmark/download/postprocess/steps.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import Any, List
+
+from shelfmark.core.logger import setup_logger
+
+from .types import PlanStep
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def record_step(steps: List[PlanStep], name: str, **details: Any) -> None:
+    steps.append(PlanStep(name=name, details=details))
+
+
+def log_plan_steps(task_id: str, steps: List[PlanStep]) -> None:
+    if not steps:
+        return
+    summary = " -> ".join(step.name for step in steps)
+    logger.debug("Processing plan for %s: %s", task_id, summary)

shelfmark/download/postprocess/transfer.py

@@ -0,0 +1,399 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import shelfmark.core.config as core_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.naming import (
+    assign_part_numbers,
+    build_library_path,
+    parse_naming_template,
+    same_filesystem,
+    sanitize_filename,
+)
+from shelfmark.core.utils import is_audiobook as check_audiobook
+from shelfmark.download.fs import atomic_copy, atomic_hardlink, atomic_move
+from shelfmark.download.postprocess.policy import get_file_organization, get_template
+
+from .scan import collect_directory_files, scan_directory_tree
+from .types import TransferPlan
+from .workspace import safe_cleanup_path
+
+logger = setup_logger("shelfmark.download.postprocess.pipeline")
+
+
+def should_hardlink(task: DownloadTask) -> bool:
+    """Check if hardlinking is enabled for this task (Prowlarr torrents only)."""
+
+    if task.source != "prowlarr":
+        return False
+
+    if not task.original_download_path:
+        return False
+
+    is_audiobook = check_audiobook(task.content_type)
+    key = "HARDLINK_TORRENTS_AUDIOBOOK" if is_audiobook else "HARDLINK_TORRENTS"
+
+    hardlink_enabled = core_config.config.get(key)
+    if hardlink_enabled is None:
+        hardlink_enabled = core_config.config.get("TORRENT_HARDLINK", False)
+
+    return bool(hardlink_enabled)
+
+
+
+def build_metadata_dict(task: DownloadTask) -> dict:
+    return {
+        "Author": task.author,
+        "Title": task.title,
+        "Subtitle": task.subtitle,
+        "Year": task.year,
+        "Series": task.series_name,
+        "SeriesPosition": task.series_position,
+    }
+
+
+def resolve_hardlink_source(
+    temp_file: Path,
+    task: DownloadTask,
+    destination: Optional[Path],
+    status_callback=None,
+) -> TransferPlan:
+    """Resolve hardlink eligibility and source path for transfers."""
+
+    use_hardlink = False
+    source_path = temp_file
+    hardlink_enabled = should_hardlink(task)
+
+    if hardlink_enabled and task.original_download_path:
+        hardlink_source = Path(task.original_download_path)
+        if destination and hardlink_source.exists() and same_filesystem(hardlink_source, destination):
+            use_hardlink = True
+            source_path = hardlink_source
+        elif hardlink_source.exists():
+            logger.warning(
+                f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
+                "Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
+            )
+            if status_callback:
+                status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
+
+    return TransferPlan(
+        source_path=source_path,
+        use_hardlink=use_hardlink,
+        allow_archive_extraction=not hardlink_enabled,
+        hardlink_enabled=hardlink_enabled,
+    )
+
+
+def is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
+    """Check if source is the torrent client path (needs copy to preserve seeding)."""
+
+    if not task.original_download_path:
+        return False
+
+    original_path = Path(task.original_download_path)
+    try:
+        return source_path.resolve() == original_path.resolve()
+    except (OSError, ValueError):
+        try:
+            return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path))
+        except Exception:
+            return False
+
+
+def _max_attempts_for_batch(file_count: int, default: int = 100) -> int:
+    if file_count <= 1:
+        return default
+    return max(default, file_count + default)
+
+
+def _transfer_single_file(
+    source_path: Path,
+    dest_path: Path,
+    use_hardlink: bool,
+    is_torrent: bool,
+    preserve_source: bool = False,
+    max_attempts: int = 100,
+) -> Tuple[Path, str]:
+    if use_hardlink:
+        final_path = atomic_hardlink(source_path, dest_path, max_attempts=max_attempts)
+        try:
+            if os.stat(source_path).st_ino == os.stat(final_path).st_ino:
+                return final_path, "hardlink"
+        except OSError:
+            return final_path, "hardlink"
+        return final_path, "copy"
+
+    if is_torrent or preserve_source:
+        return atomic_copy(source_path, dest_path, max_attempts=max_attempts), "copy"
+
+    return atomic_move(source_path, dest_path, max_attempts=max_attempts), "move"
+
+
+def transfer_book_files(
+    book_files: List[Path],
+    destination: Path,
+    task: DownloadTask,
+    use_hardlink: bool,
+    is_torrent: bool,
+    preserve_source: bool = False,
+    organization_mode: Optional[str] = None,
+) -> Tuple[List[Path], Optional[str]]:
+    if not book_files:
+        return [], "No book files found"
+
+    is_audiobook = check_audiobook(task.content_type)
+    organization_mode = organization_mode or get_file_organization(is_audiobook)
+    max_attempts = _max_attempts_for_batch(len(book_files))
+
+    final_paths: List[Path] = []
+
+    if organization_mode == "organize":
+        template = get_template(is_audiobook, "organize")
+        metadata = build_metadata_dict(task)
+
+        if len(book_files) == 1:
+            source_file = book_files[0]
+            ext = source_file.suffix.lstrip(".") or task.format or ""
+            dest_path = build_library_path(str(destination), template, metadata, extension=ext or None)
+            dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+            final_path, op = _transfer_single_file(
+                source_file,
+                dest_path,
+                use_hardlink,
+                is_torrent,
+                preserve_source=preserve_source,
+                max_attempts=max_attempts,
+            )
+            final_paths.append(final_path)
+            logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
+        else:
+            zero_pad_width = max(len(str(len(book_files))), 2)
+            files_with_parts = assign_part_numbers(book_files, zero_pad_width)
+
+            for source_file, part_number in files_with_parts:
+                ext = source_file.suffix.lstrip(".") or task.format or ""
+                file_metadata = {**metadata, "PartNumber": part_number}
+                dest_path = build_library_path(str(destination), template, file_metadata, extension=ext or None)
+                dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+                final_path, op = _transfer_single_file(
+                    source_file,
+                    dest_path,
+                    use_hardlink,
+                    is_torrent,
+                    preserve_source=preserve_source,
+                    max_attempts=max_attempts,
+                )
+                final_paths.append(final_path)
+                logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
+
+        return final_paths, None
+
+    for book_file in book_files:
+        if len(book_files) == 1 and organization_mode != "none":
+            if not task.format:
+                task.format = book_file.suffix.lower().lstrip(".")
+
+            template = get_template(is_audiobook, "rename")
+            metadata = build_metadata_dict(task)
+            extension = book_file.suffix.lstrip(".") or task.format or ""
+
+            filename = parse_naming_template(template, metadata, allow_path_separators=False)
+            filename = Path(filename).name if filename else ""
+            if filename and extension:
+                filename = f"{sanitize_filename(filename)}.{extension}"
+            else:
+                filename = book_file.name
+        else:
+            filename = book_file.name
+
+        dest_path = destination / filename
+        final_path, op = _transfer_single_file(
+            book_file,
+            dest_path,
+            use_hardlink,
+            is_torrent,
+            preserve_source=preserve_source,
+            max_attempts=max_attempts,
+        )
+        final_paths.append(final_path)
+        logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
+
+    return final_paths, None
+
+
+def process_directory(
+    directory: Path,
+    ingest_dir: Path,
+    task: DownloadTask,
+    allow_archive_extraction: bool = True,
+    use_hardlink: Optional[bool] = None,
+) -> Tuple[List[Path], Optional[str]]:
+    """Process staged directory: find book files, extract archives, move to ingest."""
+
+    try:
+        is_torrent = is_torrent_source(directory, task)
+        book_files, _, cleanup_paths, error = collect_directory_files(
+            directory,
+            task,
+            allow_archive_extraction=allow_archive_extraction,
+            status_callback=None,
+            cleanup_archives=not is_torrent,
+        )
+
+        if error:
+            if not is_torrent:
+                safe_cleanup_path(directory, task)
+                for cleanup_path in cleanup_paths:
+                    safe_cleanup_path(cleanup_path, task)
+            return [], error
+
+        if use_hardlink is None:
+            use_hardlink = should_hardlink(task)
+
+        final_paths, error = transfer_book_files(
+            book_files,
+            destination=ingest_dir,
+            task=task,
+            use_hardlink=use_hardlink,
+            is_torrent=is_torrent,
+        )
+
+        if error:
+            return [], error
+
+        if not is_torrent:
+            safe_cleanup_path(directory, task)
+            for cleanup_path in cleanup_paths:
+                safe_cleanup_path(cleanup_path, task)
+
+        return final_paths, None
+
+    except Exception as exc:
+        logger.error_trace("Task %s: error processing directory %s: %s", task.task_id, directory, exc)
+        if not is_torrent_source(directory, task):
+            safe_cleanup_path(directory, task)
+        return [], str(exc)
+
+
+def transfer_file_to_library(
+    source_path: Path,
+    library_base: str,
+    template: str,
+    metadata: dict,
+    task: DownloadTask,
+    temp_file: Optional[Path],
+    status_callback,
+    use_hardlink: bool,
+) -> Optional[str]:
+    extension = source_path.suffix.lstrip(".") or task.format
+    dest_path = build_library_path(library_base, template, metadata, extension)
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+    is_torrent = is_torrent_source(source_path, task)
+    final_path, op = _transfer_single_file(
+        source_path,
+        dest_path,
+        use_hardlink,
+        is_torrent,
+        max_attempts=_max_attempts_for_batch(1),
+    )
+    logger.info(f"Library {op}: {final_path}")
+
+    if use_hardlink and temp_file and not is_torrent_source(temp_file, task):
+        safe_cleanup_path(temp_file, task)
+
+    status_callback("complete", "Complete")
+    return str(final_path)
+
+
+def transfer_directory_to_library(
+    source_dir: Path,
+    library_base: str,
+    template: str,
+    metadata: dict,
+    task: DownloadTask,
+    temp_file: Optional[Path],
+    status_callback,
+    use_hardlink: bool,
+) -> Optional[str]:
+    content_type = task.content_type.lower() if task.content_type else None
+    source_files, _, _, scan_error = scan_directory_tree(source_dir, content_type)
+    if scan_error:
+        logger.warning(scan_error)
+        status_callback("error", scan_error)
+        if temp_file:
+            safe_cleanup_path(temp_file, task)
+        return None
+
+    if not source_files:
+        logger.warning(f"No supported files in {source_dir.name}")
+        status_callback("error", "No supported file formats found")
+        if temp_file:
+            safe_cleanup_path(temp_file, task)
+        return None
+
+    base_library_path = build_library_path(library_base, template, metadata, extension=None)
+    base_library_path.parent.mkdir(parents=True, exist_ok=True)
+
+    is_torrent = is_torrent_source(source_dir, task)
+    transferred_paths: List[Path] = []
+    max_attempts = _max_attempts_for_batch(len(source_files))
+
+    if len(source_files) == 1:
+        source_file = source_files[0]
+        ext = source_file.suffix.lstrip(".")
+        dest_path = base_library_path.with_suffix(f".{ext}")
+        final_path, op = _transfer_single_file(
+            source_file,
+            dest_path,
+            use_hardlink,
+            is_torrent,
+            max_attempts=max_attempts,
+        )
+        logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
+        transferred_paths.append(final_path)
+    else:
+        zero_pad_width = max(len(str(len(source_files))), 2)
+        files_with_parts = assign_part_numbers(source_files, zero_pad_width)
+
+        for source_file, part_number in files_with_parts:
+            ext = source_file.suffix.lstrip(".")
+            file_metadata = {**metadata, "PartNumber": part_number}
+            file_path = build_library_path(library_base, template, file_metadata, extension=ext)
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            final_path, op = _transfer_single_file(
+                source_file,
+                file_path,
+                use_hardlink,
+                is_torrent,
+                max_attempts=max_attempts,
+            )
+            logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
+            transferred_paths.append(final_path)
+
+    if use_hardlink:
+        operation = "hardlinks"
+    elif is_torrent:
+        operation = "copies"
+    else:
+        operation = "files"
+    logger.info(f"Created {len(transferred_paths)} library {operation} in {base_library_path.parent}")
+
+    if use_hardlink and temp_file and not is_torrent_source(temp_file, task):
+        safe_cleanup_path(temp_file, task)
+    elif not is_torrent:
+        safe_cleanup_path(temp_file, task)
+        safe_cleanup_path(source_dir, task)
+
+    message = f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
+    status_callback("complete", message)
+
+    return str(transferred_paths[0])