Temporarily pin Flask>=3.1.0 (#628)

- Add admin config for self-settings options visibility. Remove delivery
preferences or notifications from the view.
- Add option to use Booklore's Bookdrop API destination instead of a
specific library
- Add download path options for all torrent clients

.github/workflows/build-and-publish-docker-image.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository_owner }}/shelfmark
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -20,15 +20,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - suffix: ""
-            target: cwa-bd
-            image_name_suffix: ""
-          - suffix: "-tor"
-            target: cwa-bd-tor
-            image_name_suffix: "-tor"
-          - suffix: "-extbp"
-            target: cwa-bd-extbp
-            image_name_suffix: "-extbp"
+          - target: shelfmark
+          - target: shelfmark-lite
+            image_name_suffix: "-lite"
     steps:
       - name: Get current date
         id: date
@@ -80,4 +74,71 @@ jobs:
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
+
+  # Create aliases for backwards compatibility
+  create-aliases:
+    needs: build-and-push-images
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    permissions:
+      contents: read
+      packages: write
+    env:
+      # Legacy name for backwards compatibility (hardcoded so it works after rename)
+      LEGACY_NAME: calibre-web-automated-book-downloader
+    steps:
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create legacy aliases
+        run: |
+          # Current image names (follows repo name)
+          STANDARD="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          LITE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-lite"
+
+          # Legacy image names (hardcoded for backwards compatibility)
+          LEGACY="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.LEGACY_NAME }}"
+          LEGACY_TOR="${LEGACY}-tor"
+          LEGACY_EXTBP="${LEGACY}-extbp"
+
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+
+          # Helper function to create alias with all standard tags
+          create_alias() {
+            local SOURCE=$1
+            local ALIAS=$2
+
+            # Always create SHA tag
+            docker buildx imagetools create -t "${ALIAS}:sha-${SHA_SHORT}" "${SOURCE}:sha-${SHA_SHORT}"
+
+            if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+              VERSION="${{ github.ref_name }}"
+              VERSION_NUM="${VERSION#v}"
+              MINOR="${VERSION_NUM%.*}"
+
+              docker buildx imagetools create -t "${ALIAS}:latest" "${SOURCE}:latest"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION_NUM}" "${SOURCE}:${VERSION_NUM}"
+              docker buildx imagetools create -t "${ALIAS}:${MINOR}" "${SOURCE}:${MINOR}"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION}" "${SOURCE}:${VERSION}"
+            else
+              docker buildx imagetools create -t "${ALIAS}:dev" "${SOURCE}:dev"
+            fi
+          }
+
+          # Create legacy aliases pointing to current images
+          # calibre-web-automated-book-downloader → standard image
+          create_alias "${STANDARD}" "${LEGACY}"
+
+          # calibre-web-automated-book-downloader-tor → standard image
+          create_alias "${STANDARD}" "${LEGACY_TOR}"
+
+          # calibre-web-automated-book-downloader-extbp → lite image
+          create_alias "${LITE}" "${LEGACY_EXTBP}"

.gitignore

@@ -227,3 +227,9 @@ pyrightconfig.json
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python
 /downloaded_files
+/.local/
+*.local.*
+AGENTS.md
+.claude/
+.playwright-mcp/
+frontend-dist/

.vscode/launch.json

@@ -2,17 +2,17 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Current CWABD File",
+      "name": "Python Debugger: Current Shelfmark File",
       "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false,
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader",
-        "TEMP_DIR": "/tmp/cwa-book-downloader",
+        "INGEST_DIR": "/tmp/shelfmark",
+        "TEMP_DIR": "/tmp/shelfmark",
         "LOG_LEVEL": "DEBUG",
-        "LOG_ROOT": "/tmp/cwa-book-downloader",
+        "LOG_ROOT": "/tmp/shelfmark",
         "ENABLE_LOGGING": "true",
         "DOCKERMODE": "false",
         "DEBUG": "true",
@@ -27,7 +27,7 @@
       "preLaunchTask": "docker-compose up (dev)", // Spin up dev containers
       "postDebugTask": "docker-compose down (dev)", // Optional: tear them down
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -38,7 +38,7 @@
       "preLaunchTask": "docker-compose up (prod)",
       "postDebugTask": "docker-compose down (prod)",
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -54,9 +54,9 @@
   ],
   "compounds": [
     {
-      "name": "Launch CWA-BD",
+      "name": "Launch Shelfmark",
       "configurations": [
-        "Launch cwa-bd app.py",
+        "Launch Shelfmark app.py",
         "Launch Browser"
       ]
     }

Dockerfile

@@ -14,8 +14,9 @@ WORKDIR /frontend
 # Copy frontend package files
 COPY src/frontend/package*.json ./
 
-# Install dependencies
-RUN npm ci
+# Install dependencies (cache mount for faster rebuilds)
+RUN --mount=type=cache,target=/root/.npm \
+    npm ci
 
 # Copy frontend source
 COPY src/frontend/ ./
@@ -44,9 +45,9 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    NAME=Calibre-Web-Automated-Book-Downloader \
+    NAME=Shelfmark \
     PYTHONPATH=/app \
-    # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
+    # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
     LC_ALL=en_US.UTF-8
@@ -67,7 +68,14 @@ RUN apt-get update && \
     # For debug
     zip iputils-ping \
     # For user switching
-    sudo && \
+    sudo \
+    # --- Tor support (activated via USING_TOR=true) ---
+    tor \
+    supervisor \
+    iptables && \
+    # Configure iptables alternatives for tor.sh compatibility
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
     # Cleanup APT cache *after* all installs in this layer
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
@@ -84,12 +92,11 @@ RUN apt-get update && \
 WORKDIR /app
 
 # Install Python dependencies using pip
-# Upgrade pip first, then copy requirements and install
-# Copying requirements-base.txt separately leverages build cache
-COPY requirements-base.txt .
-RUN pip install --no-cache-dir -r requirements-base.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
+# Copying requirements files separately leverages build cache
+# Cache mount persists pip cache between builds for faster installs
+COPY requirements-base.txt requirements-shelfmark.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-base.txt
 
 # Copy application code *after* dependencies are installed
 COPY . .
@@ -100,7 +107,7 @@ COPY --from=frontend-builder /frontend/dist /app/frontend-dist
 # Final setup: permissions and directories in one layer
 # Only creating directories and setting executable bits.
 # Ownership will be handled by the entrypoint script.
-RUN mkdir -p /var/log/cwa-book-downloader /cwa-book-ingest && \
+RUN mkdir -p /var/log/shelfmark /books && \
     chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
 
 # Expose the application port
@@ -115,7 +122,7 @@ HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 
-FROM base AS cwa-bd
+FROM base AS shelfmark
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -123,49 +130,34 @@ RUN apt-get update && \
     xvfb \
     # For screen recording
     ffmpeg \
-    # --- Chromium ---
+    # --- Chromium (unpinned - uses latest from Debian repos) ---
+    # Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
+    # This flag is set in internal_bypasser.py _get_browser_args()
     chromium \
-    # --- ChromeDriver ---
-    chromium-driver \
+    chromium-common \
     # For tkinter (pyautogui)
-    python3-tk
-
-# install additional dependencies
-COPY requirements-cwa-bd.txt .
-RUN pip install --no-cache-dir -r requirements-cwa-bd.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
-
-# Add this line to grant read/execute permissions to others
-RUN chmod -R o+rx /usr/bin/chromium && \
-    chmod -R o+rx /usr/bin/chromedriver && \
-    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
-
-# Default command to run the application entrypoint script
-CMD ["/app/entrypoint.sh"]
-
-FROM cwa-bd AS cwa-bd-tor
-
-ENV USING_TOR=true
-
-# Install Tor and dependencies
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    # --- Tor ---
-    tor \
-    # --- iptables ---
-    iptables && \
-    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
-    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
-    # Cleanup APT cache *after* all installs in this layer
+    python3-tk \
+    # For RAR extraction
+    unrar-free && \
+    # Create symlink so rarfile library can find unrar
+    ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
+    # Cleanup APT cache
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Override the default command to run Tor
+# Install additional dependencies (requirements file already copied in base stage)
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-shelfmark.txt
+
+# Grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium && \
+    chmod -R o+rwx /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+
+# Default command to run the application entrypoint script
 CMD ["/app/entrypoint.sh"]
 
-FROM base AS cwa-bd-extbp
+FROM base AS shelfmark-lite
 
 ENV USING_EXTERNAL_BYPASSER=true
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install dev build preview typecheck clean up down docker-build refresh
+.PHONY: help install dev build preview typecheck clean up down docker-build refresh restart
 
 # Frontend directory
 FRONTEND_DIR := src/frontend
@@ -21,6 +21,7 @@ help:
 	@echo "Backend (Docker):"
 	@echo "  up         - Start backend services"
 	@echo "  down       - Stop backend services"
+	@echo "  restart    - Restart backend services (no rebuild)"
 	@echo "  docker-build - Build Docker image"
 	@echo "  refresh    - Rebuild and restart backend services"
 
@@ -70,6 +71,11 @@ docker-build:
 	@echo "Building Docker image..."
 	docker compose -f $(COMPOSE_FILE) build
 
+# Restart backend services (no rebuild)
+restart:
+	@echo "Restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) restart
+
 # Rebuild and restart backend services
 refresh:
 	@echo "Rebuilding and restarting backend services..."

app.py

@@ -1,825 +0,0 @@
-"""Flask web application for book download service with URL rewrite support."""
-
-import logging
-import io, re, os
-import sqlite3
-import time
-from datetime import datetime, timedelta
-from functools import wraps
-from flask import Flask, request, jsonify, send_file, send_from_directory, session
-from flask_cors import CORS
-from flask_socketio import SocketIO, emit
-from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.security import check_password_hash
-from werkzeug.wrappers import Response
-import typing
-
-from logger import setup_logger
-from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE, SUPPORTED_FORMATS
-from env import FLASK_HOST, FLASK_PORT, CWA_DB_PATH, DEBUG, USING_EXTERNAL_BYPASSER, BUILD_VERSION, RELEASE_VERSION, CALIBRE_WEB_URL
-import backend
-
-from models import SearchFilters
-from websocket_manager import ws_manager
-
-logger = setup_logger(__name__)
-app = Flask(__name__)
-app.wsgi_app = ProxyFix(app.wsgi_app)  # type: ignore
-app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0  # Disable caching
-app.config['APPLICATION_ROOT'] = '/'
-
-# Determine async mode based on DEBUG setting
-# In production (DEBUG=False) with Gunicorn + gevent worker, use 'gevent'
-# In development (DEBUG=True) with Flask dev server, use 'threading'
-if DEBUG:
-    async_mode = 'threading'
-else:
-    async_mode = 'gevent'
-
-# Initialize Flask-SocketIO with reverse proxy support
-socketio = SocketIO(
-    app,
-    cors_allowed_origins="*",
-    async_mode=async_mode,
-    logger=False,
-    engineio_logger=False,
-    # Reverse proxy / Traefik compatibility settings
-    path='/socket.io',
-    ping_timeout=60,  # Time to wait for pong response
-    ping_interval=25,  # Send ping every 25 seconds
-    # Allow both websocket and polling for better compatibility
-    transports=['websocket', 'polling'],
-    # Enable CORS for all origins (you can restrict this in production)
-    allow_upgrades=True,
-    # Important for proxies that buffer
-    http_compression=True
-)
-
-# Initialize WebSocket manager
-ws_manager.init_app(app, socketio)
-logger.info(f"Flask-SocketIO initialized with async_mode='{async_mode}'")
-
-# Rate limiting for login attempts
-# Structure: {username: {'count': int, 'lockout_until': datetime}}
-failed_login_attempts: typing.Dict[str, typing.Dict[str, typing.Any]] = {}
-MAX_LOGIN_ATTEMPTS = 10
-LOCKOUT_DURATION_MINUTES = 30
-
-def cleanup_old_lockouts() -> None:
-    """Remove expired lockout entries to prevent memory buildup."""
-    current_time = datetime.now()
-    expired_users = [
-        username for username, data in failed_login_attempts.items()
-        if 'lockout_until' in data and data['lockout_until'] < current_time
-    ]
-    for username in expired_users:
-        logger.info(f"Lockout expired for user: {username}")
-        del failed_login_attempts[username]
-
-def is_account_locked(username: str) -> bool:
-    """Check if an account is currently locked due to failed login attempts."""
-    cleanup_old_lockouts()
-    
-    if username not in failed_login_attempts:
-        return False
-    
-    lockout_until = failed_login_attempts[username].get('lockout_until')
-    if lockout_until and datetime.now() < lockout_until:
-        return True
-    
-    return False
-
-def record_failed_login(username: str, ip_address: str) -> bool:
-    """
-    Record a failed login attempt and lock account if threshold is reached.
-    Returns True if account is now locked, False otherwise.
-    """
-    if username not in failed_login_attempts:
-        failed_login_attempts[username] = {'count': 0}
-    
-    failed_login_attempts[username]['count'] += 1
-    count = failed_login_attempts[username]['count']
-    
-    logger.warning(f"Failed login attempt {count}/{MAX_LOGIN_ATTEMPTS} for user '{username}' from IP {ip_address}")
-    
-    if count >= MAX_LOGIN_ATTEMPTS:
-        lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES)
-        failed_login_attempts[username]['lockout_until'] = lockout_until
-        logger.warning(f"Account locked for user '{username}' until {lockout_until.strftime('%Y-%m-%d %H:%M:%S')} due to {count} failed login attempts")
-        return True
-    
-    return False
-
-def clear_failed_logins(username: str) -> None:
-    """Clear failed login attempts for a user after successful login."""
-    if username in failed_login_attempts:
-        del failed_login_attempts[username]
-        logger.debug(f"Cleared failed login attempts for user: {username}")
-
-# Enable CORS in development mode for local frontend development
-if DEBUG:
-    CORS(app, resources={
-        r"/*": {
-            "origins": ["http://localhost:5173", "http://127.0.0.1:5173"],
-            "supports_credentials": True,
-            "allow_headers": ["Content-Type", "Authorization"],
-            "methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"]
-        }
-    })
-
-# Custom log filter to exclude routine status endpoint polling
-class StatusEndpointFilter(logging.Filter):
-    """Filter out routine status endpoint requests to reduce log noise."""
-    def filter(self, record):
-        # Exclude GET /api/status requests
-        if hasattr(record, 'getMessage'):
-            message = record.getMessage()
-            if 'GET /api/status' in message:
-                return False
-        return True
-
-# Flask logger
-app.logger.handlers = logger.handlers
-app.logger.setLevel(logger.level)
-# Also handle Werkzeug's logger
-werkzeug_logger = logging.getLogger('werkzeug')
-werkzeug_logger.handlers = logger.handlers
-werkzeug_logger.setLevel(logger.level)
-# Add filter to suppress routine status endpoint polling logs
-werkzeug_logger.addFilter(StatusEndpointFilter())
-
-# Set up authentication defaults
-# The secret key will reset every time we restart, which will
-# require users to authenticate again
-
-# Session cookie security - set to 'true' if exclusively using HTTPS
-session_cookie_secure_env = os.getenv('SESSION_COOKIE_SECURE', 'false').lower()
-SESSION_COOKIE_SECURE = session_cookie_secure_env in ['true', 'yes', '1']
-
-app.config.update(
-    SECRET_KEY = os.urandom(64),
-    SESSION_COOKIE_HTTPONLY = True,
-    SESSION_COOKIE_SAMESITE = 'Lax',
-    SESSION_COOKIE_SECURE = SESSION_COOKIE_SECURE,
-    PERMANENT_SESSION_LIFETIME = 604800  # 7 days in seconds
-)
-
-logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {session_cookie_secure_env})")
-
-def login_required(f):
-    @wraps(f)
-    def decorated_function(*args, **kwargs):
-        # If the CWA_DB_PATH variable exists, but isn't a valid
-        # path, return a server error
-        if CWA_DB_PATH is not None and not os.path.isfile(CWA_DB_PATH):
-            logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path")
-            return jsonify({"error": "Internal Server Error"}), 500
-        
-        # If no database is configured, allow access
-        if not CWA_DB_PATH:
-            return f(*args, **kwargs)
-        
-        # Check if user has a valid session
-        if 'user_id' not in session:
-            return jsonify({"error": "Unauthorized"}), 401
-        
-        return f(*args, **kwargs)
-    return decorated_function
-
-
-# Serve frontend static files
-@app.route('/assets/<path:filename>')
-def serve_frontend_assets(filename: str) -> Response:
-    """
-    Serve static assets from the built frontend.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist', 'assets'), filename)
-
-@app.route('/')
-def index() -> Response:
-    """
-    Serve the React frontend application.
-    Authentication is handled by the React app itself.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'index.html')
-
-@app.route('/logo.png')
-def logo() -> Response:
-    """
-    Serve logo from built frontend assets.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'),
-        'logo.png', mimetype='image/png')
-
-@app.route('/favicon.ico')
-@app.route('/favico<path:_>')
-def favicon(_ : typing.Any = None) -> Response:
-    """
-    Serve favicon from built frontend assets.
-    """
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'),
-        'favicon.ico', mimetype='image/vnd.microsoft.icon')
-
-from typing import Union, Tuple
-
-if DEBUG:
-    import subprocess
-    import time
-    if USING_EXTERNAL_BYPASSER:
-        STOP_GUI = lambda: None  # No-op for external bypasser
-    else:
-        from cloudflare_bypasser import _reset_driver as STOP_GUI
-    @app.route('/debug', methods=['GET'])
-    @login_required
-    def debug() -> Union[Response, Tuple[Response, int]]:
-        """
-        This will run the /app/debug.sh script, which will generate a debug zip with all the logs
-        The file will be named /tmp/cwa-book-downloader-debug.zip
-        And then return it to the user
-        """
-        try:
-            # Run the debug script
-            STOP_GUI()
-            time.sleep(1)
-            result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True)
-            if result.returncode != 0:
-                raise Exception(f"Debug script failed: {result.stderr}")
-            logger.info(f"Debug script executed: {result.stdout}")
-            debug_file_path = result.stdout.strip().split('\n')[-1]
-            if not os.path.exists(debug_file_path):
-                logger.error("Debug zip file not found after running debug script")
-                return jsonify({"error": "Failed to generate debug information"}), 500
-                
-            # Return the file to the user
-            return send_file(
-                debug_file_path,
-                mimetype='application/zip',
-                download_name=os.path.basename(debug_file_path),
-                as_attachment=True
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}")
-            return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500
-        except Exception as e:
-            logger.error_trace(f"Debug endpoint error: {e}")
-            return jsonify({"error": str(e)}), 500
-
-if DEBUG:
-    @app.route('/api/restart', methods=['GET'])
-    @login_required
-    def restart() -> Union[Response, Tuple[Response, int]]:
-        """
-        Restart the application
-        """
-        os._exit(0)
-
-@app.route('/api/search', methods=['GET'])
-@login_required
-def api_search() -> Union[Response, Tuple[Response, int]]:
-    """
-    Search for books matching the provided query.
-
-    Query Parameters:
-        query (str): Search term (ISBN, title, author, etc.)
-        isbn (str): Book ISBN
-        author (str): Book Author
-        title (str): Book Title
-        lang (str): Book Language
-        sort (str): Order to sort results
-        content (str): Content type of book
-        format (str): File format filter (pdf, epub, mobi, azw3, fb2, djvu, cbz, cbr)
-
-    Returns:
-        flask.Response: JSON array of matching books or error response.
-    """
-    query = request.args.get('query', '')
-
-    filters = SearchFilters(
-        isbn = request.args.getlist('isbn'),
-        author = request.args.getlist('author'),
-        title = request.args.getlist('title'),
-        lang = request.args.getlist('lang'),
-        sort = request.args.get('sort'),
-        content = request.args.getlist('content'),
-        format = request.args.getlist('format'),
-    )
-
-    if not query and not any(vars(filters).values()):
-        return jsonify([])
-
-    try:
-        books = backend.search_books(query, filters)
-        return jsonify(books)
-    except Exception as e:
-        logger.error_trace(f"Search error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/info', methods=['GET'])
-@login_required
-def api_info() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get detailed book information.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON object with book details, or an error message.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        book = backend.get_book_info(book_id)
-        if book:
-            return jsonify(book)
-        return jsonify({"error": "Book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Info error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download', methods=['GET'])
-@login_required
-def api_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Queue a book for download.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON status object indicating success or failure.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        priority = int(request.args.get('priority', 0))
-        success = backend.queue_book(book_id, priority)
-        if success:
-            return jsonify({"status": "queued", "priority": priority})
-        return jsonify({"error": "Failed to queue book"}), 500
-    except Exception as e:
-        logger.error_trace(f"Download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/config', methods=['GET'])
-@login_required
-def api_config() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get application configuration for frontend.
-    """
-    try:
-        config = {
-            "calibre_web_url": CALIBRE_WEB_URL,
-            "debug": DEBUG,
-            "build_version": BUILD_VERSION,
-            "release_version": RELEASE_VERSION,
-            "book_languages": _SUPPORTED_BOOK_LANGUAGE,
-            "default_language": BOOK_LANGUAGE,
-            "supported_formats": SUPPORTED_FORMATS
-        }
-        return jsonify(config)
-    except Exception as e:
-        logger.error_trace(f"Config error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/health', methods=['GET'])
-def api_health() -> Union[Response, Tuple[Response, int]]:
-    """
-    Health check endpoint for container orchestration.
-    No authentication required.
-    
-    Returns:
-        flask.Response: JSON with status "ok".
-    """
-    return jsonify({"status": "ok"})
-
-@app.route('/api/status', methods=['GET'])
-@login_required
-def api_status() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current download queue status.
-
-    Returns:
-        flask.Response: JSON object with queue status.
-    """
-    try:
-        status = backend.queue_status()
-        return jsonify(status)
-    except Exception as e:
-        logger.error_trace(f"Status error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/localdownload', methods=['GET'])
-@login_required
-def api_local_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Download an EPUB file from local storage if available.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: The EPUB file if found, otherwise an error response.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        file_data, book_info = backend.get_book_data(book_id)
-        if file_data is None:
-            # Book data not found or not available
-            return jsonify({"error": "File not found"}), 404
-        # Santize the file name
-        file_name = book_info.title
-        file_name = re.sub(r'[\\/:*?"<>|]', '_', file_name.strip())[:245]
-        file_extension = book_info.format
-        # Prepare the file for sending to the client
-        data = io.BytesIO(file_data)
-        return send_file(
-            data,
-            download_name=f"{file_name}.{file_extension}",
-            as_attachment=True
-        )
-
-    except Exception as e:
-        logger.error_trace(f"Local download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download/<book_id>/cancel', methods=['DELETE'])
-@login_required
-def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Cancel a download.
-
-    Path Parameters:
-        book_id (str): Book identifier to cancel
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        success = backend.cancel_download(book_id)
-        if success:
-            return jsonify({"status": "cancelled", "book_id": book_id})
-        return jsonify({"error": "Failed to cancel download or book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Cancel download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/<book_id>/priority', methods=['PUT'])
-@login_required
-def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Set priority for a queued book.
-
-    Path Parameters:
-        book_id (str): Book identifier
-
-    Request Body:
-        priority (int): New priority level (lower number = higher priority)
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'priority' not in data:
-            return jsonify({"error": "Priority not provided"}), 400
-            
-        priority = int(data['priority'])
-        success = backend.set_book_priority(book_id, priority)
-        
-        if success:
-            return jsonify({"status": "updated", "book_id": book_id, "priority": priority})
-        return jsonify({"error": "Failed to update priority or book not found"}), 404
-    except ValueError:
-        return jsonify({"error": "Invalid priority value"}), 400
-    except Exception as e:
-        logger.error_trace(f"Set priority error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/reorder', methods=['POST'])
-@login_required
-def api_reorder_queue() -> Union[Response, Tuple[Response, int]]:
-    """
-    Bulk reorder queue by setting new priorities.
-
-    Request Body:
-        book_priorities (dict): Mapping of book_id to new priority
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'book_priorities' not in data:
-            return jsonify({"error": "book_priorities not provided"}), 400
-            
-        book_priorities = data['book_priorities']
-        if not isinstance(book_priorities, dict):
-            return jsonify({"error": "book_priorities must be a dictionary"}), 400
-            
-        # Validate all priorities are integers
-        for book_id, priority in book_priorities.items():
-            if not isinstance(priority, int):
-                return jsonify({"error": f"Invalid priority for book {book_id}"}), 400
-                
-        success = backend.reorder_queue(book_priorities)
-        
-        if success:
-            return jsonify({"status": "reordered", "updated_count": len(book_priorities)})
-        return jsonify({"error": "Failed to reorder queue"}), 500
-    except Exception as e:
-        logger.error_trace(f"Reorder queue error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/order', methods=['GET'])
-@login_required
-def api_queue_order() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current queue order for display.
-
-    Returns:
-        flask.Response: JSON array of queued books with their order and priorities.
-    """
-    try:
-        queue_order = backend.get_queue_order()
-        return jsonify({"queue": queue_order})
-    except Exception as e:
-        logger.error_trace(f"Queue order error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/downloads/active', methods=['GET'])
-@login_required
-def api_active_downloads() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get list of currently active downloads.
-
-    Returns:
-        flask.Response: JSON array of active download book IDs.
-    """
-    try:
-        active_downloads = backend.get_active_downloads()
-        return jsonify({"active_downloads": active_downloads})
-    except Exception as e:
-        logger.error_trace(f"Active downloads error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/clear', methods=['DELETE'])
-@login_required
-def api_clear_completed() -> Union[Response, Tuple[Response, int]]:
-    """
-    Clear all completed, errored, or cancelled books from tracking.
-
-    Returns:
-        flask.Response: JSON with count of removed books.
-    """
-    try:
-        removed_count = backend.clear_completed()
-        
-        # Broadcast status update after clearing
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(backend.queue_status())
-        
-        return jsonify({"status": "cleared", "removed_count": removed_count})
-    except Exception as e:
-        logger.error_trace(f"Clear completed error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.errorhandler(404)
-def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 404 (Not Found) errors.
-
-    Args:
-        error (HTTPException): The 404 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 404 status.
-    """
-    logger.warning(f"404 error: {request.url} : {error}")
-    return jsonify({"error": "Resource not found"}), 404
-
-@app.errorhandler(500)
-def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 500 (Internal Server) errors.
-
-    Args:
-        error (HTTPException): The 500 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 500 status.
-    """
-    logger.error_trace(f"500 error: {error}")
-    return jsonify({"error": "Internal server error"}), 500
-
-@app.route('/api/auth/login', methods=['POST'])
-def api_login() -> Union[Response, Tuple[Response, int]]:
-    """
-    Login endpoint that validates credentials and creates a session.
-    Includes rate limiting: 10 failed attempts = 30 minute lockout.
-    
-    Request Body:
-        username (str): Username
-        password (str): Password
-        remember_me (bool): Whether to extend session duration
-        
-    Returns:
-        flask.Response: JSON with success status or error message.
-    """
-    try:
-        # Get client IP address (handles reverse proxy forwarding)
-        ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
-        if ip_address and ',' in ip_address:
-            # X-Forwarded-For can contain multiple IPs, take the first one
-            ip_address = ip_address.split(',')[0].strip()
-        
-        data = request.get_json()
-        if not data:
-            return jsonify({"error": "No data provided"}), 400
-            
-        username = data.get('username', '').strip()
-        password = data.get('password', '')
-        remember_me = data.get('remember_me', False)
-        
-        if not username or not password:
-            return jsonify({"error": "Username and password are required"}), 400
-        
-        # Check if account is locked due to failed login attempts
-        if is_account_locked(username):
-            lockout_until = failed_login_attempts[username].get('lockout_until')
-            remaining_time = (lockout_until - datetime.now()).total_seconds() / 60
-            logger.warning(f"Login attempt blocked for locked account '{username}' from IP {ip_address}")
-            return jsonify({
-                "error": f"Account temporarily locked due to multiple failed login attempts. Try again in {int(remaining_time)} minutes."
-            }), 429
-        
-        # If the database doesn't exist, authentication always succeeds
-        if not CWA_DB_PATH:
-            session['user_id'] = username
-            session.permanent = remember_me
-            clear_failed_logins(username)
-            logger.info(f"Login successful for user '{username}' from IP {ip_address} (no DB configured)")
-            return jsonify({"success": True})
-        
-        # If the CWA_DB_PATH variable exists, but isn't a valid path, return error
-        if not os.path.isfile(CWA_DB_PATH):
-            logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path")
-            return jsonify({"error": "Database configuration error"}), 500
-        
-        # Validate credentials against database
-        try:
-            db_path = os.fspath(CWA_DB_PATH)
-            db_uri = f"file:{db_path}?mode=ro&immutable=1"
-            conn = sqlite3.connect(db_uri, uri=True)
-            cur = conn.cursor()
-            cur.execute("SELECT password FROM user WHERE name = ?", (username,))
-            row = cur.fetchone()
-            conn.close()
-            
-            # Check if user exists and password is correct
-            if not row or not row[0] or not check_password_hash(row[0], password):
-                # Record failed login attempt
-                is_now_locked = record_failed_login(username, ip_address)
-                
-                if is_now_locked:
-                    return jsonify({
-                        "error": f"Account locked due to {MAX_LOGIN_ATTEMPTS} failed login attempts. Try again in {LOCKOUT_DURATION_MINUTES} minutes."
-                    }), 429
-                else:
-                    attempts_remaining = MAX_LOGIN_ATTEMPTS - failed_login_attempts[username]['count']
-                    # Only show attempts remaining when 5 or fewer attempts remain (after 6+ failed attempts)
-                    if attempts_remaining <= 5:
-                        return jsonify({
-                            "error": f"Invalid username or password. {attempts_remaining} attempts remaining."
-                        }), 401
-                    else:
-                        return jsonify({
-                            "error": "Invalid username or password."
-                        }), 401
-            
-            # Successful authentication - create session and clear failed attempts
-            session['user_id'] = username
-            session.permanent = remember_me
-            clear_failed_logins(username)
-            logger.info(f"Login successful for user '{username}' from IP {ip_address} (remember_me={remember_me})")
-            return jsonify({"success": True})
-            
-        except Exception as e:
-            logger.error_trace(f"Database error during login: {e}")
-            return jsonify({"error": "Authentication system error"}), 500
-            
-    except Exception as e:
-        logger.error_trace(f"Login error: {e}")
-        return jsonify({"error": "Login failed"}), 500
-
-@app.route('/api/auth/logout', methods=['POST'])
-def api_logout() -> Union[Response, Tuple[Response, int]]:
-    """
-    Logout endpoint that clears the session.
-    
-    Returns:
-        flask.Response: JSON with success status.
-    """
-    try:
-        # Get client IP address (handles reverse proxy forwarding)
-        ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
-        if ip_address and ',' in ip_address:
-            ip_address = ip_address.split(',')[0].strip()
-        
-        username = session.get('user_id', 'unknown')
-        session.clear()
-        logger.info(f"Logout successful for user '{username}' from IP {ip_address}")
-        return jsonify({"success": True})
-    except Exception as e:
-        logger.error_trace(f"Logout error: {e}")
-        return jsonify({"error": "Logout failed"}), 500
-
-@app.route('/api/auth/check', methods=['GET'])
-def api_auth_check() -> Union[Response, Tuple[Response, int]]:
-    """
-    Check if user has a valid session.
-    
-    Returns:
-        flask.Response: JSON with authentication status and whether auth is required.
-    """
-    try:
-        # If no database is configured, authentication is not required
-        if not CWA_DB_PATH:
-            return jsonify({
-                "authenticated": True,
-                "auth_required": False
-            })
-        
-        # Check if user has a valid session
-        is_authenticated = 'user_id' in session
-        return jsonify({
-            "authenticated": is_authenticated,
-            "auth_required": True
-        })
-    except Exception as e:
-        logger.error_trace(f"Auth check error: {e}")
-        return jsonify({
-            "authenticated": False,
-            "auth_required": True
-        })
-
-# Catch-all route for React Router (must be last)
-# This handles client-side routing by serving index.html for any unmatched routes
-@app.route('/<path:path>')
-def catch_all(path: str) -> Response:
-    """
-    Serve the React app for any route not matched by API endpoints.
-    This allows React Router to handle client-side routing.
-    Authentication is handled by the React app itself.
-    """
-    # If the request is for an API endpoint or static file, let it 404
-    if path.startswith('api/') or path.startswith('assets/'):
-        return jsonify({"error": "Resource not found"}), 404
-    # Otherwise serve the React app
-    return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'index.html')
-
-# WebSocket event handlers
-@socketio.on('connect')
-def handle_connect():
-    """Handle client connection."""
-    logger.info("WebSocket client connected")
-    # Send initial status to the newly connected client
-    try:
-        status = backend.queue_status()
-        emit('status_update', status)
-    except Exception as e:
-        logger.error(f"Error sending initial status: {e}")
-
-@socketio.on('disconnect')
-def handle_disconnect():
-    """Handle client disconnection."""  
-    logger.info("WebSocket client disconnected")
-
-@socketio.on('request_status')
-def handle_status_request():
-    """Handle manual status request from client."""
-    try:
-        status = backend.queue_status()
-        emit('status_update', status)
-    except Exception as e:
-        logger.error(f"Error handling status request: {e}")
-        emit('error', {'message': 'Failed to get status'})
-
-logger.log_resource_usage()
-
-if __name__ == '__main__':
-    logger.info(f"Starting Flask application with WebSocket support on {FLASK_HOST}:{FLASK_PORT} (debug={DEBUG})")
-    socketio.run(
-        app,
-        host=FLASK_HOST,
-        port=FLASK_PORT,
-        debug=DEBUG,
-        allow_unsafe_werkzeug=True  # For development only
-    )

backend.py

@@ -1,426 +0,0 @@
-"""Backend logic for the book download application."""
-
-import threading, time
-import shutil
-from pathlib import Path
-from typing import Dict, List, Optional, Any, Tuple
-import subprocess
-import os
-from concurrent.futures import ThreadPoolExecutor, Future
-from threading import Event
-
-from logger import setup_logger
-from config import CUSTOM_SCRIPT
-from env import (INGEST_DIR, DOWNLOAD_PATHS, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE,
-                 MAX_CONCURRENT_DOWNLOADS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL)
-from models import book_queue, BookInfo, QueueStatus, SearchFilters
-import book_manager
-
-logger = setup_logger(__name__)
-
-# Import WebSocket manager (will be initialized by app.py)
-try:
-    from websocket_manager import ws_manager
-except ImportError:
-    logger.warning("WebSocket manager not available")
-    ws_manager = None
-
-def _sanitize_filename(filename: str) -> str:
-    """Sanitize a filename by replacing spaces with underscores and removing invalid characters."""
-    keepcharacters = (' ','.','_')
-    return "".join(c for c in filename if c.isalnum() or c in keepcharacters).rstrip()
-
-def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
-    """Search for books matching the query.
-    
-    Args:
-        query: Search term
-        filters: Search filters object
-        
-    Returns:
-        List[Dict]: List of book information dictionaries
-    """
-    try:
-        books = book_manager.search_books(query, filters)
-        return [_book_info_to_dict(book) for book in books]
-    except Exception as e:
-        logger.error_trace(f"Error searching books: {e}")
-        return []
-
-def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
-    """Get detailed information for a specific book.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Optional[Dict]: Book information dictionary if found
-    """
-    try:
-        book = book_manager.get_book_info(book_id)
-        return _book_info_to_dict(book)
-    except Exception as e:
-        logger.error_trace(f"Error getting book info: {e}")
-        return None
-
-def queue_book(book_id: str, priority: int = 0) -> bool:
-    """Add a book to the download queue with specified priority.
-    
-    Args:
-        book_id: Book identifier
-        priority: Priority level (lower number = higher priority)
-        
-    Returns:
-        bool: True if book was successfully queued
-    """
-    try:
-        book_info = book_manager.get_book_info(book_id)
-        book_queue.add(book_id, book_info, priority)
-        logger.info(f"Book queued with priority {priority}: {book_info.title}")
-        
-        # Broadcast status update via WebSocket
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-        
-        return True
-    except Exception as e:
-        logger.error_trace(f"Error queueing book: {e}")
-        return False
-
-def queue_status() -> Dict[str, Dict[str, Any]]:
-    """Get current status of the download queue.
-    
-    Returns:
-        Dict: Queue status organized by status type with serialized book data
-    """
-    status = book_queue.get_status()
-    for _, books in status.items():
-        for _, book_info in books.items():
-            if book_info.download_path:
-                if not os.path.exists(book_info.download_path):
-                    book_info.download_path = None
-
-    # Convert Enum keys to strings and BookInfo objects to dicts for JSON serialization
-    return {
-        status_type.value: {
-            book_id: _book_info_to_dict(book_info)
-            for book_id, book_info in books.items()
-        }
-        for status_type, books in status.items()
-    }
-
-def get_book_data(book_id: str) -> Tuple[Optional[bytes], BookInfo]:
-    """Get book data for a specific book, including its title.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Tuple[Optional[bytes], str]: Book data if available, and the book title
-    """
-    try:
-        book_info = book_queue._book_data[book_id]
-        path = book_info.download_path
-        with open(path, "rb") as f:
-            return f.read(), book_info
-    except Exception as e:
-        logger.error_trace(f"Error getting book data: {e}")
-        if book_info:
-            book_info.download_path = None
-        return None, book_info if book_info else BookInfo(id=book_id, title="Unknown")
-
-def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
-    """Convert BookInfo object to dictionary representation."""
-    return {
-        key: value for key, value in book.__dict__.items()
-        if value is not None
-    }
-
-def _prepare_download_folder(book_info: BookInfo) -> Path:
-    """Prepare final content-type subdir"""
-    content = book_info.content
-    content_dir = DOWNLOAD_PATHS.get(content) if content and content in DOWNLOAD_PATHS else INGEST_DIR
-    os.makedirs(content_dir, exist_ok=True)
-    return content_dir
-
-def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Optional[str]:
-    """Download and process a book with cancellation support.
-    
-    Args:
-        book_id: Book identifier
-        cancel_flag: Threading event to signal cancellation
-        
-    Returns:
-        str: Path to the downloaded book if successful, None otherwise
-    """
-    try:
-        # Check for cancellation before starting
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before starting: {book_id}")
-            return None
-            
-        book_info = book_queue._book_data[book_id]
-        logger.info(f"Starting download: {book_info.title}")
-
-        if USE_BOOK_TITLE:
-            book_name = _sanitize_filename(book_info.title)
-        else:
-            book_name = book_id
-        # If format is not set, use the format of the first download URL
-        if book_info.format == "":
-            book_info.format = book_info.download_urls[0].split(".")[-1]
-        book_name += f".{book_info.format}"
-        book_path = TMP_DIR / book_name
-
-        # Check cancellation before download
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before book manager call: {book_id}")
-            return None
-        
-        progress_callback = lambda progress: update_download_progress(book_id, progress)
-        status_callback = lambda status: update_download_status(book_id, status)
-        success_download_url = book_manager.download_book(book_info, book_path, progress_callback, cancel_flag, status_callback)
-        
-        # Stop progress updates
-        cancel_flag.wait(0.1)  # Brief pause for progress thread cleanup
-        
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during download: {book_id}")
-            # Clean up partial download
-            if book_path.exists():
-                book_path.unlink()
-            return None
-            
-        if not success_download_url:
-            raise Exception("Unknown error downloading book")
-
-        # Check cancellation before post-processing
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before post-processing: {book_id}")
-            if book_path.exists():
-                book_path.unlink()
-            return None
-
-        # Update status to verifying
-        book_queue.update_status(book_id, QueueStatus.VERIFYING)
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-        logger.info(f"Verifying download: {book_info.title}")
-
-        if CUSTOM_SCRIPT:
-            logger.info(f"Running custom script: {CUSTOM_SCRIPT}")
-            subprocess.run([CUSTOM_SCRIPT, book_path])
-            
-        # Update status to ingesting
-        book_queue.update_status(book_id, QueueStatus.INGESTING)
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-        
-        if success_download_url and book_info.format == "":
-            book_info.format = success_download_url.split(".")[-1]
-            book_name += f".{book_info.format}"
-
-        final_dir = _prepare_download_folder(book_info)
-        intermediate_path = final_dir / f"{book_id}.crdownload"
-        final_path = final_dir / book_name
-        
-        if os.path.exists(book_path):
-            logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}")
-            try:
-                shutil.move(book_path, intermediate_path)
-            except Exception as e:
-                try:
-                    logger.debug(f"Error moving book: {e}, will try copying instead")
-                    shutil.move(book_path, intermediate_path)
-                except Exception as e:
-                    logger.debug(f"Error copying book: {e}, will try copying without permissions instead")
-                    shutil.copyfile(book_path, intermediate_path)
-                os.remove(book_path)
-            
-            # Final cancellation check before completing
-            if cancel_flag.is_set():
-                logger.info(f"Download cancelled before final rename: {book_id}")
-                if intermediate_path.exists():
-                    intermediate_path.unlink()
-                return None
-                
-            os.rename(intermediate_path, final_path)
-            logger.info(f"Download completed successfully: {book_info.title}")
-            
-        return str(final_path)
-    except Exception as e:
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during error handling: {book_id}")
-        else:
-            logger.error_trace(f"Error downloading book: {e}")
-        return None
-
-def update_download_progress(book_id: str, progress: float) -> None:
-    """Update download progress."""
-    book_queue.update_progress(book_id, progress)
-    
-    # Broadcast progress via WebSocket
-    if ws_manager and ws_manager.is_enabled():
-        ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
-
-def update_download_status(book_id: str, status: str) -> None:
-    """Update download status."""
-    # Map string status to QueueStatus enum
-    status_map = {
-        'queued': QueueStatus.QUEUED,
-        'resolving': QueueStatus.RESOLVING,
-        'bypassing': QueueStatus.BYPASSING,
-        'downloading': QueueStatus.DOWNLOADING,
-        'verifying': QueueStatus.VERIFYING,
-        'ingesting': QueueStatus.INGESTING,
-        'complete': QueueStatus.COMPLETE,
-        'available': QueueStatus.AVAILABLE,
-        'error': QueueStatus.ERROR,
-        'done': QueueStatus.DONE,
-        'cancelled': QueueStatus.CANCELLED,
-    }
-    
-    queue_status_enum = status_map.get(status.lower())
-    if queue_status_enum:
-        book_queue.update_status(book_id, queue_status_enum)
-        
-        # Broadcast status update via WebSocket
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-
-def cancel_download(book_id: str) -> bool:
-    """Cancel a download.
-    
-    Args:
-        book_id: Book identifier to cancel
-        
-    Returns:
-        bool: True if cancellation was successful
-    """
-    result = book_queue.cancel_download(book_id)
-    
-    # Broadcast status update via WebSocket
-    if result and ws_manager and ws_manager.is_enabled():
-        ws_manager.broadcast_status_update(queue_status())
-    
-    return result
-
-def set_book_priority(book_id: str, priority: int) -> bool:
-    """Set priority for a queued book.
-    
-    Args:
-        book_id: Book identifier
-        priority: New priority level (lower = higher priority)
-        
-    Returns:
-        bool: True if priority was successfully changed
-    """
-    return book_queue.set_priority(book_id, priority)
-
-def reorder_queue(book_priorities: Dict[str, int]) -> bool:
-    """Bulk reorder queue.
-    
-    Args:
-        book_priorities: Dict mapping book_id to new priority
-        
-    Returns:
-        bool: True if reordering was successful
-    """
-    return book_queue.reorder_queue(book_priorities)
-
-def get_queue_order() -> List[Dict[str, any]]:
-    """Get current queue order for display."""
-    return book_queue.get_queue_order()
-
-def get_active_downloads() -> List[str]:
-    """Get list of currently active downloads."""
-    return book_queue.get_active_downloads()
-
-def clear_completed() -> int:
-    """Clear all completed downloads from tracking."""
-    return book_queue.clear_completed()
-
-def _process_single_download(book_id: str, cancel_flag: Event) -> None:
-    """Process a single download job."""
-    try:
-        # Status will be updated through callbacks during download process
-        # (resolving -> bypassing -> downloading -> verifying -> ingesting -> complete)
-        download_path = _download_book_with_cancellation(book_id, cancel_flag)
-        
-        if cancel_flag.is_set():
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-            # Broadcast cancellation
-            if ws_manager and ws_manager.is_enabled():
-                ws_manager.broadcast_status_update(queue_status())
-            return
-            
-        if download_path:
-            book_queue.update_download_path(book_id, download_path)
-            new_status = QueueStatus.COMPLETE
-        else:
-            new_status = QueueStatus.ERROR
-            
-        book_queue.update_status(book_id, new_status)
-        
-        # Broadcast final status (completed or error)
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-        
-        logger.info(
-            f"Book {book_id} download {'successful' if download_path else 'failed'}"
-        )
-        
-    except Exception as e:
-        if not cancel_flag.is_set():
-            logger.error_trace(f"Error in download processing: {e}")
-            book_queue.update_status(book_id, QueueStatus.ERROR)
-        else:
-            logger.info(f"Download cancelled: {book_id}")
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-        
-        # Broadcast error/cancelled status
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-
-def concurrent_download_loop() -> None:
-    """Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
-    logger.info(f"Starting concurrent download loop with {MAX_CONCURRENT_DOWNLOADS} workers")
-    
-    with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_DOWNLOADS, thread_name_prefix="BookDownload") as executor:
-        active_futures: Dict[Future, str] = {}  # Track active download futures
-        
-        while True:
-            # Clean up completed futures
-            completed_futures = [f for f in active_futures if f.done()]
-            for future in completed_futures:
-                book_id = active_futures.pop(future)
-                try:
-                    future.result()  # This will raise any exceptions from the worker
-                except Exception as e:
-                    logger.error_trace(f"Future exception for {book_id}: {e}")
-            
-            # Start new downloads if we have capacity
-            while len(active_futures) < MAX_CONCURRENT_DOWNLOADS:
-                next_download = book_queue.get_next()
-                if not next_download:
-                    break
-                    
-                book_id, cancel_flag = next_download
-                logger.info(f"Starting concurrent download: {book_id}")
-                
-                # Submit download job to thread pool
-                future = executor.submit(_process_single_download, book_id, cancel_flag)
-                active_futures[future] = book_id
-            
-            # Brief sleep to prevent busy waiting
-            time.sleep(MAIN_LOOP_SLEEP_TIME)
-
-# Start concurrent download coordinator
-download_coordinator_thread = threading.Thread(
-    target=concurrent_download_loop,
-    daemon=True,
-    name="DownloadCoordinator"
-)
-download_coordinator_thread.start()
-
-logger.info(f"Download system initialized with {MAX_CONCURRENT_DOWNLOADS} concurrent workers")

book_manager.py

@@ -1,495 +0,0 @@
-"""Book download manager handling search and retrieval operations."""
-
-import time, json, os, re
-from pathlib import Path
-from urllib.parse import quote
-from typing import List, Optional, Dict, Union, Callable
-from threading import Event
-from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet
-
-import downloader
-from logger import setup_logger
-from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL
-from env import AA_DONATOR_KEY, USE_CF_BYPASS, PRIORITIZE_WELIB, ALLOW_USE_WELIB, DOWNLOAD_PATHS
-from models import BookInfo, SearchFilters
-logger = setup_logger(__name__)
-
-
-
-def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
-    """Search for books matching the query.
-
-    Args:
-        query: Search term (ISBN, title, author, etc.)
-
-    Returns:
-        List[BookInfo]: List of matching books
-
-    Raises:
-        Exception: If no books found or parsing fails
-    """
-    query_html = quote(query)
-
-    if filters.isbn:
-        # ISBNs are included in query string
-        isbns = " || ".join(
-            [f"('isbn13:{isbn}' || 'isbn10:{isbn}')" for isbn in filters.isbn]
-        )
-        query_html = quote(f"({isbns}) {query}")
-
-    filters_query = ""
-
-    for value in filters.lang or BOOK_LANGUAGE:
-        if value != "all":
-            filters_query += f"&lang={quote(value)}"
-
-    if filters.sort:
-        filters_query += f"&sort={quote(filters.sort)}"
-
-    if filters.content:
-        for value in filters.content:
-            filters_query += f"&content={quote(value)}"
-
-    # Handle format filter
-    formats_to_use = filters.format if filters.format else SUPPORTED_FORMATS
-
-    index = 1
-    for filter_type, filter_values in vars(filters).items():
-        if filter_type == "author" or filter_type == "title" and filter_values:
-            for value in filter_values:
-                filters_query += (
-                    f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}"
-                )
-                index += 1
-
-    url = (
-        f"{AA_BASE_URL}"
-        f"/search?index=&page=1&display=table"
-        f"&acc=aa_download&acc=external_download"
-        f"&ext={'&ext='.join(formats_to_use)}"
-        f"&q={query_html}"
-        f"{filters_query}"
-    )
-
-    html = downloader.html_get_page(url)
-    if not html:
-        raise Exception("Failed to fetch search results")
-
-    if "No files found." in html:
-        logger.info(f"No books found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    soup = BeautifulSoup(html, "html.parser")
-    tbody: Tag | NavigableString | None = soup.find("table")
-
-    if not tbody:
-        logger.warning(f"No results table found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    books = []
-    if isinstance(tbody, Tag):
-        for line_tr in tbody.find_all("tr"):
-            try:
-                book = _parse_search_result_row(line_tr)
-                if book:
-                    books.append(book)
-            except Exception as e:
-                logger.error_trace(f"Failed to parse search result row: {e}")
-
-    books.sort(
-        key=lambda x: (
-            SUPPORTED_FORMATS.index(x.format)
-            if x.format in SUPPORTED_FORMATS
-            else len(SUPPORTED_FORMATS)
-        )
-    )
-
-    return books
-
-
-def _parse_search_result_row(row: Tag) -> Optional[BookInfo]:
-    """Parse a single search result row into a BookInfo object."""
-    try:
-        # Skip ad rows
-        if row.text.strip().lower().startswith("your ad here"):
-            return None
-        cells = row.find_all("td")
-        preview_img = cells[0].find("img")
-        preview = preview_img["src"] if preview_img else None
-
-        return BookInfo(
-            id=row.find_all("a")[0]["href"].split("/")[-1],
-            preview=preview,
-            title=cells[1].find("span").next,
-            author=cells[2].find("span").next,
-            publisher=cells[3].find("span").next,
-            year=cells[4].find("span").next,
-            language=cells[7].find("span").next,
-            content=cells[8].find("span").next.lower(),
-            format=cells[9].find("span").next.lower(),
-            size=cells[10].find("span").next,
-        )
-    except Exception as e:
-        logger.error_trace(f"Error parsing search result row: {e}")
-        return None
-
-
-def get_book_info(book_id: str) -> BookInfo:
-    """Get detailed information for a specific book.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-
-    Returns:
-        BookInfo: Detailed book information
-    """
-    url = f"{AA_BASE_URL}/md5/{book_id}"
-    html = downloader.html_get_page(url)
-
-    if not html:
-        raise Exception(f"Failed to fetch book info for ID: {book_id}")
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    return _parse_book_info_page(soup, book_id)
-
-
-def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
-    """Parse the book info page HTML into a BookInfo object."""
-    data = soup.select_one("body > main > div:nth-of-type(1)")
-
-    if not data:
-        raise Exception(f"Failed to parse book info for ID: {book_id}")
-
-    preview: str = ""
-
-    node = data.select_one("div:nth-of-type(1) > img")
-    if node:
-        preview_value = node.get("src", "")
-        if isinstance(preview_value, list):
-            preview = preview_value[0]
-        else:
-            preview = preview_value
-
-    data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
-    divs = list(data.children)
-
-    every_url = soup.find_all("a")
-    slow_urls_no_waitlist = set()
-    slow_urls_with_waitlist = set()
-    external_urls_libgen = set()
-    external_urls_z_lib = set()
-    external_urls_welib = set()
-
-    for url in every_url:
-        try:
-            if url.text.strip().lower().startswith("slow partner server"):
-                if (
-                    url.next is not None
-                    and url.next.next is not None
-                    and "waitlist" in url.next.next.strip().lower()
-                ):
-                    internal_text = url.next.next.strip().lower()
-                    if "no waitlist" in internal_text:
-                        slow_urls_no_waitlist.add(url["href"])
-                    else:
-                        slow_urls_with_waitlist.add(url["href"])
-            elif (
-                url.next is not None
-                and url.next.next is not None
-                and "click “GET” at the top" in url.next.next.text.strip()
-            ):
-                libgen_url = url["href"]
-                # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
-                libgen_url = re.sub(r'libgen\.(lc|is|bz|st)', 'libgen.gl', url["href"])
-
-                external_urls_libgen.add(libgen_url)
-            elif url.text.strip().lower().startswith("z-lib"):
-                if ".onion/" not in url["href"]:
-                    external_urls_z_lib.add(url["href"])
-        except:
-            pass
-
-    external_urls_welib = _get_download_urls_from_welib(book_id) if USE_CF_BYPASS else set()
-
-    urls = []
-    urls += list(external_urls_welib) if PRIORITIZE_WELIB else []
-    urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
-    urls += list(external_urls_libgen)
-    urls += list(external_urls_welib) if not PRIORITIZE_WELIB else []
-    urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
-    urls += list(external_urls_z_lib)
-
-    for i in range(len(urls)):
-        urls[i] = downloader.get_absolute_url(AA_BASE_URL, urls[i])
-
-    # Remove empty urls
-    urls = [url for url in urls if url != ""]
-
-    # Filter out divs that are not text
-    original_divs = divs
-    divs = [div for div in divs if div.text.strip() != ""]
-
-    all_details = _find_in_divs(divs, " · ")
-    format = ""
-    size = ""
-    content = ""
-    
-    for _details in all_details:
-        _details = _details.split(" · ")
-        for f in _details:
-            if format == "" and f.strip().lower() in SUPPORTED_FORMATS:
-                format = f.strip().lower()
-            if size == "" and any(u in f.strip().lower() for u in ["mb", "kb", "gb"]):
-                size = f.strip().lower()
-            if content == "":
-                for ct in DOWNLOAD_PATHS.keys():
-                    if ct in f.strip().lower():
-                        content = ct
-                        break
-        if format == "" or size == "":
-            for f in _details:
-                stripped = f.strip().lower()
-                if format == "" and stripped and " " not in stripped:
-                    format = stripped
-                if size == "" and "." in stripped:
-                    size = stripped
-    
-    book_title = _find_in_divs(divs, "🔍")[0].strip("🔍").strip()
-
-    # Extract basic information
-    description = _extract_book_description(soup)
-
-    book_info = BookInfo(
-        id=book_id,
-        preview=preview,
-        title=book_title,
-        content=content,
-        publisher=_find_in_divs(divs, "icon-[mdi--company]", isClass=True)[0],
-        author=_find_in_divs(divs, "icon-[mdi--user-edit]", isClass=True)[0],
-        format=format,
-        size=size,
-        description=description,
-        download_urls=urls,
-    )
-
-    # Extract additional metadata
-    info = _extract_book_metadata(original_divs[-6])
-    book_info.info = info
-
-    # Set language and year from metadata if available
-    if info.get("Language"):
-        book_info.language = info["Language"][0]
-    if info.get("Year"):
-        book_info.year = info["Year"][0]
-
-    # TODO :
-    # Backfill missing metadata from original book
-    # To do this, we need to cache the results of search_books() in some kind of LRU
-
-    return book_info
-
-def _find_in_divs(divs: List[str], text: str, isClass: bool = False) -> List[str]:
-    divs_found = []
-    for div in divs:
-        if isClass:
-            if div.find(class_ = text):
-                divs_found.append(div.text.strip())
-        else:
-            if text in div.text.strip():
-                divs_found.append(div.text.strip())
-    return divs_found
-
-def _get_download_urls_from_welib(book_id: str) -> set[str]:
-    if ALLOW_USE_WELIB == False:
-        return set()
-    """Get download urls from welib.org."""
-    url = f"https://welib.org/md5/{book_id}"
-    logger.info(f"Getting download urls from welib.org for {book_id}. While this uses the bypasser, it will not start downloading them yet.")
-    html = downloader.html_get_page(url, use_bypasser=True)
-    if not html:
-        return []
-    soup = BeautifulSoup(html, "html.parser")
-    download_links = soup.find_all("a", href=True)
-    download_links = [link["href"] for link in download_links]
-    download_links = [link for link in download_links if "/slow_download/" in link]
-    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
-    return set(download_links)
-
-def _get_next_value_div(label_div: Tag) -> Optional[Tag]:
-    """Find the next sibling div that holds the value for a metadata label."""
-    sibling = label_div.next_sibling
-    while sibling:
-        if isinstance(sibling, Tag) and sibling.name == "div":
-            return sibling
-        sibling = sibling.next_sibling
-    return None
-
-def _extract_book_description(soup: BeautifulSoup) -> Optional[str]:
-    """Extract the primary or alternative description from the book page."""
-    container = soup.select_one(".js-md5-top-box-description")
-    if not container:
-        return None
-
-    description: Optional[str] = None
-    alternative: Optional[str] = None
-
-    label_divs = container.select("div.text-xs.text-gray-500.uppercase")
-    for label_div in label_divs:
-        label_text = label_div.get_text(strip=True).lower()
-        value_div = _get_next_value_div(label_div)
-        if not value_div:
-            continue
-
-        value_text = value_div.get_text(separator=" ", strip=True)
-        if not value_text:
-            continue
-
-        if label_text == "description":
-            return value_text
-        if label_text == "alternative description" and not alternative:
-            alternative = value_text
-
-    if alternative:
-        return alternative
-
-    # Fallback to the first text block inside the description container
-    fallback_div = container.find("div", class_="mb-1")
-    if fallback_div:
-        fallback_text = fallback_div.get_text(separator=" ", strip=True)
-        if fallback_text:
-            return fallback_text
-
-    return None
-
-def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]:
-    """Extract metadata from book info divs."""
-    info: Dict[str, List[str]] = {}
-
-    # Process the first set of metadata
-    sub_datas = metadata_divs.find_all("div")[0]
-    sub_datas = list(sub_datas.children)
-    for sub_data in sub_datas:
-        if sub_data.text.strip() == "":
-            continue
-        sub_data = list(sub_data.children)
-        key = sub_data[0].text.strip()
-        value = sub_data[1].text.strip()
-        if key not in info:
-            info[key] = set()
-        info[key].add(value)
-    
-    # make set into list
-    for key, value in info.items():
-        info[key] = list(value)
-
-    # Filter relevant metadata
-    relevant_prefixes = [
-        "ISBN-",
-        "ALTERNATIVE",
-        "ASIN",
-        "Goodreads",
-        "Language",
-        "Year",
-    ]
-    return {
-        k.strip(): v
-        for k, v in info.items()
-        if any(k.lower().startswith(prefix.lower()) for prefix in relevant_prefixes)
-        and "filename" not in k.lower()
-    }
-
-
-def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str], None]] = None) -> Optional[str]:
-    """Download a book from available sources.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-        title: Book title for logging
-        progress_callback: Optional callback for download progress updates
-        cancel_flag: Optional cancellation flag
-        status_callback: Optional callback for status updates
-
-    Returns:
-        str: Download URL if successful, None otherwise
-    """
-
-    if len(book_info.download_urls) == 0:
-        book_info = get_book_info(book_info.id)
-    download_links = book_info.download_urls
-
-    # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources.
-    if AA_DONATOR_KEY != "":
-        download_links.insert(
-            0,
-            f"{AA_BASE_URL}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}",
-        )
-
-    for link in download_links:
-        try:
-            # Update status to resolving before attempting download URL fetch
-            if status_callback:
-                status_callback("resolving")
-
-            download_url = _get_download_url(link, book_info.title, cancel_flag, status_callback)
-            if download_url != "":
-                # Update status to downloading before starting actual download
-                if status_callback:
-                    status_callback("downloading")
-
-                logger.info(f"Downloading `{book_info.title}` from `{download_url}`")
-
-                data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag)
-                if not data:
-                    raise Exception("No data received")
-
-                logger.info(f"Download finished. Writing to {book_path}")
-                with open(book_path, "wb") as f:
-                    f.write(data.getbuffer())
-                logger.info(f"Writing `{book_info.title}` successfully")
-                return download_url
-
-        except Exception as e:
-            logger.error_trace(f"Failed to download from {link}: {e}")
-            continue
-
-    return None
-
-
-def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str], None]] = None) -> str:
-    """Extract actual download URL from various source pages."""
-
-    url = ""
-
-    if link.startswith(f"{AA_BASE_URL}/dyn/api/fast_download.json"):
-        page = downloader.html_get_page(link, status_callback=status_callback)
-        url = json.loads(page).get("download_url")
-    else:
-        html = downloader.html_get_page(link, status_callback=status_callback)
-
-        if html == "":
-            return ""
-
-        soup = BeautifulSoup(html, "html.parser")
-
-        if link.startswith("https://z-lib."):
-            download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
-            if download_link:
-                url = download_link[0]["href"]
-        elif "/slow_download/" in link:
-            download_links = soup.find_all("a", href=True, string="📚 Download now")
-            if not download_links:
-                countdown = soup.find_all("span", class_="js-partner-countdown")
-                if countdown:
-                    sleep_time = int(countdown[0].text)
-                    logger.info(f"Waiting {sleep_time}s for {title}")
-                    if cancel_flag is not None and cancel_flag.wait(timeout=sleep_time):
-                        logger.info(f"Cancelled wait for {title}")
-                        return ""
-                    url = _get_download_url(link, title, cancel_flag, status_callback)
-            else:
-                url = download_links[0]["href"]
-        else:
-            url = soup.find_all("a", string="GET")[0]["href"]
-
-    return downloader.get_absolute_url(link, url)

cloudflare_bypasser.py

@@ -1,491 +0,0 @@
-import time
-import os
-import socket
-from urllib.parse import urlparse
-import threading
-import env
-from env import LOG_DIR, DEBUG
-import signal
-from datetime import datetime
-import subprocess
-import requests
-from typing import Optional
-
-# --- SeleniumBase Import ---
-from seleniumbase import Driver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.common.exceptions import TimeoutException
-
-import network
-from logger import setup_logger
-from env import MAX_RETRY, DEFAULT_SLEEP
-from config import PROXIES, CUSTOM_DNS, DOH_SERVER, VIRTUAL_SCREEN_SIZE, RECORDING_DIR
-
-logger = setup_logger(__name__)
-network.init()
-
-DRIVER = None
-DISPLAY = {
-    "xvfb": None,
-    "ffmpeg": None,
-}
-LAST_USED = None
-LOCKED = threading.Lock()
-TENTATIVE_CURRENT_URL = None
-
-def _reset_pyautogui_display_state():
-    try:
-        import pyautogui
-        import Xlib.display
-        pyautogui._pyautogui_x11._display = (
-                    Xlib.display.Display(os.environ['DISPLAY'])
-                )
-    except Exception as e:
-        logger.warning(f"Error resetting pyautogui display state: {e}")
-
-def _is_bypassed(sb, escape_emojis : bool = True) -> bool:
-    """Enhanced bypass detection with more comprehensive checks"""
-    try:
-        # Get page information with error handling
-        try:
-            title = sb.get_title().lower()
-        except:
-            title = ""
-            
-        try:
-            body = sb.get_text("body").lower()
-        except:
-            body = ""
-            
-        try:
-            current_url = sb.get_current_url()
-        except:
-            current_url = ""
-        
-        # Check if page is too long, if so we are probably bypassed
-        if len(body.strip()) > 100000:
-            logger.debug(f"Page content too long, we are probably bypassed len: {len(body.strip())}")
-            return True
-        
-        # Detect if there is an emoji in the page, any utf8 emoji, if so we are probably bypassed
-        if escape_emojis:
-            import emoji
-            emoji_list = emoji.emoji_list(body)
-            if len(emoji_list) >= 3:
-                logger.debug(f"Detected emoji in page, we are probably bypassed len: {len(emoji_list)}")
-                return True
-
-        # Enhanced verification texts for newer Cloudflare versions
-        verification_texts = [
-            "just a moment",
-            "verify you are human",
-            "verifying you are human",
-            "cloudflare.com/products/turnstile/?utm_source=turnstile"
-        ]
-        
-        # Check for Cloudflare indicators
-        for text in verification_texts:
-            if text in title or text in body:
-                logger.debug(f"Cloudflare indicator found: '{text}' in page")
-                return False
-        
-        # Additional checks for specific Cloudflare patterns
-        if "cf-" in body or "cloudflare" in current_url.lower():
-            logger.debug("Cloudflare patterns detected in page")
-            return False
-            
-        # Check if we're still on a challenge page (common Cloudflare pattern)
-        if "/cdn-cgi/" in current_url:
-            logger.debug("Still on Cloudflare CDN challenge page")
-            return False
-            
-        # If page is mostly empty, it might still be loading
-        if len(body.strip()) < 50:
-            logger.debug("Page content too short, might still be loading")
-            return False
-            
-        logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {len(body)}")
-        return True
-        
-    except Exception as e:
-        logger.warning(f"Error checking bypass status: {e}")
-        # If we can't check, assume we're not bypassed
-        return False
-
-def _bypass_method_1(sb) -> bool:
-    """Original bypass method using uc_gui_click_captcha"""
-    try:
-        logger.debug("Attempting bypass method 1: uc_gui_click_captcha")
-        sb.uc_gui_click_captcha()
-        time.sleep(3)
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 1 failed on first try: {e}")
-        try:
-            time.sleep(5)
-            sb.wait_for_element_visible('body', timeout=10)
-            sb.uc_gui_click_captcha()
-            time.sleep(3)
-            return _is_bypassed(sb)
-        except Exception as e2:
-            logger.debug(f"Method 1 failed on second try: {e2}")
-            try:
-                time.sleep(DEFAULT_SLEEP)
-                sb.uc_gui_click_captcha()
-                time.sleep(5)
-                return _is_bypassed(sb)
-            except Exception as e3:
-                logger.debug(f"Method 1 completely failed: {e3}")
-                return False
-
-def _bypass_method_2(sb) -> bool:
-    """Alternative bypass method using longer waits and manual interaction"""
-    try:
-        logger.debug("Attempting bypass method 2: wait and reload")
-        # Wait longer for page to load completely
-        time.sleep(10)
-        
-        # Try refreshing the page
-        sb.refresh()
-        time.sleep(8)
-        
-        # Check if bypass worked after refresh
-        if _is_bypassed(sb):
-            return True
-            
-        # Try clicking on the page center (sometimes helps trigger bypass)
-        try:
-            sb.click_if_visible("body", timeout=5)
-            time.sleep(5)
-        except:
-            pass
-            
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 2 failed: {e}")
-        return False
-
-def _bypass_method_3(sb) -> bool:
-    """Third bypass method using user-agent rotation and stealth mode"""
-    try:
-        logger.debug("Attempting bypass method 3: stealth approach")
-        # Wait a random amount to appear more human
-        import random
-        wait_time = random.uniform(8, 15)
-        time.sleep(wait_time)
-        
-        # Try to scroll the page (human-like behavior)
-        try:
-            sb.scroll_to_bottom()
-            time.sleep(2)
-            sb.scroll_to_top()
-            time.sleep(3)
-        except:
-            pass
-            
-        # Check if this helped
-        if _is_bypassed(sb):
-            return True
-            
-        # Try the original captcha click as last resort
-        try:
-            sb.uc_gui_click_captcha()
-            time.sleep(5)
-        except:
-            pass
-            
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 3 failed: {e}")
-        return False
-
-def _bypass(sb, max_retries: int = MAX_RETRY) -> None:
-    """Enhanced bypass function with multiple strategies"""
-    try_count = 0
-    methods = [_bypass_method_1, _bypass_method_2, _bypass_method_3]
-
-    while not _is_bypassed(sb):
-        if try_count >= max_retries:
-            logger.warning("Exceeded maximum retries. Bypass failed.")
-            break
-            
-        method_index = try_count % len(methods)
-        method = methods[method_index]
-        
-        logger.info(f"Bypass attempt {try_count + 1} / {max_retries} using {method.__name__}")
-        
-        try_count += 1
-
-        # Progressive backoff: wait longer between retries
-        wait_time = min(DEFAULT_SLEEP * (try_count - 1), 15)
-        if wait_time > 0:
-            logger.info(f"Waiting {wait_time}s before trying...")
-            time.sleep(wait_time)
-
-        try:
-            if method(sb):
-                logger.info(f"Bypass successful using {method.__name__}")
-                return
-        except Exception as e:
-            logger.warning(f"Exception in {method.__name__}: {e}")
-
-        logger.info(f"Bypass method {method.__name__} failed.")
-
-def _get_chromium_args():
-    
-    arguments = [
-        # Ignore certificate and SSL errors (similar to curl's --insecure)
-        "--ignore-certificate-errors",
-        "--ignore-ssl-errors",
-        "--allow-running-insecure-content",
-        "--ignore-certificate-errors-spki-list",
-        "--ignore-certificate-errors-skip-list"
-    ]
-    
-    # Conditionally add verbose logging arguments
-    if DEBUG:
-        arguments.extend([
-            "--enable-logging", # Enable Chrome browser logging
-            "--v=1",            # Set verbosity level for Chrome logs
-            "--log-file=" + str(LOG_DIR / "chrome_browser.log")
-        ])
-
-    # Add proxy settings if configured
-    if PROXIES:
-        proxy_url = PROXIES.get('https') or PROXIES.get('http')
-        if proxy_url:
-            arguments.append(f'--proxy-server={proxy_url}')
-
-    # --- Add Custom DNS settings ---
-    try:
-        if len(CUSTOM_DNS) > 0:
-            if DOH_SERVER:
-                logger.info(f"Configuring DNS over HTTPS (DoH) with server: {DOH_SERVER}")
-
-                # TODO: This is probably broken and a halucination,
-                # but it should still default to google DOH so its fine...
-                arguments.extend(['--enable-features=DnsOverHttps', '--dns-over-https-mode=secure', f'--dns-over-https-servers="{DOH_SERVER}"'])
-                doh_hostname = urlparse(DOH_SERVER).hostname
-                if doh_hostname:
-                    try:
-                        arguments.append(f'--host-resolver-rules=MAP {doh_hostname} {socket.gethostbyname(doh_hostname)}')
-                    except socket.gaierror:
-                        logger.warning(f"Could not resolve DoH hostname: {doh_hostname}")
-            elif CUSTOM_DNS:
-                arguments.append(f'--dns-server="{",".join(CUSTOM_DNS)}"')
-                arguments.append(f'--disable-features=DnsOverHttps')
-    except Exception as e:
-        logger.error_trace(f"Error configuring DNS settings: {e}")
-    return arguments
-
-CHROMIUM_ARGS = _get_chromium_args()
-
-def _get(url, retry : int = MAX_RETRY):
-    try:
-        logger.info(f"SB_GET: {url}")
-        sb = _get_driver()
-        
-        # Enhanced page loading with better error handling
-        logger.debug("Opening URL with SeleniumBase...")
-        sb.uc_open_with_reconnect(url, DEFAULT_SLEEP)
-        time.sleep(DEFAULT_SLEEP)
-        
-        # Log current page title and URL for debugging
-        try:
-            current_url = sb.get_current_url()
-            current_title = sb.get_title()
-            logger.debug(f"Page loaded - URL: {current_url}, Title: {current_title}")
-        except Exception as debug_e:
-            logger.debug(f"Could not get page info: {debug_e}")
-        
-        # Attempt bypass
-        logger.debug("Starting bypass process...")
-        _bypass(sb)
-        
-        if _is_bypassed(sb):
-            logger.info("Bypass successful.")
-            return sb.page_source
-        else:
-            logger.warning("Bypass completed but page still shows Cloudflare protection")
-            # Log page content for debugging (truncated)
-            try:
-                page_text = sb.get_text("body")[:500] + "..." if len(sb.get_text("body")) > 500 else sb.get_text("body")
-                logger.debug(f"Page content: {page_text}")
-            except:
-                pass
-            
-    except Exception as e:
-        # Enhanced error logging with full stack trace
-        import traceback
-        error_details = f"Exception type: {type(e).__name__}, Message: {str(e)}"
-        stack_trace = traceback.format_exc()
-        
-        if retry == 0:
-            logger.error(f"Failed to initialize browser after all retries: {error_details}")
-            logger.debug(f"Full stack trace: {stack_trace}")
-            _reset_driver()
-            raise e
-        
-        logger.warning(f"Failed to bypass Cloudflare (retry {MAX_RETRY - retry + 1}/{MAX_RETRY}): {error_details}")
-        logger.debug(f"Stack trace: {stack_trace}")
-        
-        # Reset driver on certain errors
-        if "WebDriverException" in str(type(e)) or "SessionNotCreatedException" in str(type(e)):
-            logger.info("Resetting driver due to WebDriver error...")
-            _reset_driver()
-            
-    return _get(url, retry - 1)
-
-def get(url, retry : int = MAX_RETRY):
-    global LOCKED, TENTATIVE_CURRENT_URL, LAST_USED
-    with LOCKED:
-        TENTATIVE_CURRENT_URL = url
-        ret = _get(url, retry)
-        LAST_USED = time.time()
-        return ret
-
-def _init_driver():
-    global DRIVER
-    if DRIVER:
-        _reset_driver()
-    driver = Driver(uc=True, headless=False, size=f"{VIRTUAL_SCREEN_SIZE[0]},{VIRTUAL_SCREEN_SIZE[1]}", chromium_arg=CHROMIUM_ARGS)
-    DRIVER = driver
-    time.sleep(DEFAULT_SLEEP)
-    return driver
-
-def _get_driver():
-    global DRIVER, DISPLAY
-    global LAST_USED
-    logger.info("Getting driver...")
-    LAST_USED = time.time()
-    if env.DOCKERMODE and env.USE_CF_BYPASS and not DISPLAY["xvfb"]:
-        from pyvirtualdisplay import Display
-        display = Display(visible=False, size=VIRTUAL_SCREEN_SIZE)
-        display.start()
-        logger.info("Display started")
-        DISPLAY["xvfb"] = display
-        time.sleep(DEFAULT_SLEEP)
-        _reset_pyautogui_display_state()
-
-        if env.DEBUG:
-            timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
-            output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
-
-            ffmpeg_cmd = [
-                "ffmpeg",
-                "-y",
-                "-f", "x11grab",
-                "-video_size", f"{VIRTUAL_SCREEN_SIZE[0]}x{VIRTUAL_SCREEN_SIZE[1]}",
-                "-i", f":{display.display}",
-                "-c:v", "libx264",
-                "-preset", "ultrafast",  # or "veryfast" (trade speed for slightly better compression)
-                "-maxrate", "700k",      # Slightly higher bitrate for text clarity
-                "-bufsize", "1400k",    # Buffer size (2x maxrate)
-                "-crf", "36",  # Adjust as needed:  higher = smaller, lower = better quality (23 is visually lossless)
-                "-pix_fmt", "yuv420p",  # Crucial for compatibility with most players
-                "-tune", "animation",   # Optimize encoding for screen content
-                "-x264-params", "bframes=0:deblock=-1,-1", # Optimize for text, disable b-frames and deblocking
-                "-r", "15",         # Reduce frame rate (if content allows)
-                "-an",                # Disable audio recording (if not needed)
-                output_file.as_posix(),
-                "-nostats", "-loglevel", "0"
-            ]
-            logger.info("Starting FFmpeg recording to %s", output_file)
-            logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
-            DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
-    if not DRIVER:
-        return _init_driver()
-    logger.log_resource_usage()
-    return DRIVER
-
-def _reset_driver():
-    logger.log_resource_usage()
-    logger.info("Resetting driver...")
-    global DRIVER, DISPLAY
-    if DRIVER:
-        try:
-            DRIVER.quit()
-            DRIVER = None
-        except Exception as e:
-            logger.warning(f"Error quitting driver: {e}")
-        time.sleep(0.5)
-    if DISPLAY["xvfb"]:
-        try:
-            DISPLAY["xvfb"].stop()
-            DISPLAY["xvfb"] = None
-        except Exception as e:
-            logger.warning(f"Error stopping display: {e}")
-        time.sleep(0.5)
-    try:
-        os.system("pkill -f Xvfb")
-    except Exception as e:
-        logger.debug(f"Error killing Xvfb: {e}")
-    time.sleep(0.5)
-    if DISPLAY["ffmpeg"]:
-        try:
-            DISPLAY["ffmpeg"].send_signal(signal.SIGINT)
-            DISPLAY["ffmpeg"] = None
-        except Exception as e:
-            logger.debug(f"Error stopping ffmpeg: {e}")
-        time.sleep(0.5)
-    try:
-        os.system("pkill -f ffmpeg")
-    except Exception as e:
-        logger.debug(f"Error killing ffmpeg: {e}")
-    time.sleep(0.5)
-    try:
-        os.system("pkill -f chrom")
-    except Exception as e:
-        logger.debug(f"Error killing chrom: {e}")
-    time.sleep(0.5)
-    logger.info("Driver reset.")
-    logger.log_resource_usage()
-
-def _cleanup_driver():
-    global LOCKED
-    global LAST_USED
-    with LOCKED:
-        if LAST_USED:
-            if time.time() - LAST_USED >= env.BYPASS_RELEASE_INACTIVE_MIN * 60:
-                _reset_driver()
-                LAST_USED = None
-                logger.info("Driver reset due to inactivity.")
-
-def _cleanup_loop():
-    while True:
-        _cleanup_driver()
-        time.sleep(max(env.BYPASS_RELEASE_INACTIVE_MIN / 2, 1))
-
-def _init_cleanup_thread():
-    cleanup_thread = threading.Thread(target=_cleanup_loop)
-    cleanup_thread.daemon = True
-    cleanup_thread.start()
-
-def wait_for_result(func, timeout : int = 10, condition : any = True):
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        result = func()
-        if condition(result):
-            return result
-        time.sleep(0.5)
-    return None
-_init_cleanup_thread()
-
-
-def get_bypassed_page(url: str) -> Optional[str]:
-    """Fetch HTML content from a URL using the internal Cloudflare Bypasser.
-
-    Args:
-        url: Target URL
-    Returns:
-        str: HTML content if successful, None otherwise
-    """
-
-    response_html = get(url)
-    logger.debug(f"Cloudflare Bypasser response length: {len(response_html)}")
-    if response_html.strip() != "":
-        return response_html
-    else:
-        raise requests.exceptions.RequestException("Failed to bypass Cloudflare")

cloudflare_bypasser_external.py

@@ -1,34 +0,0 @@
-from logger import setup_logger
-from typing import Optional
-import requests
-
-try:
-    from env import EXT_BYPASSER_PATH, EXT_BYPASSER_TIMEOUT, EXT_BYPASSER_URL
-except ImportError:
-    raise RuntimeError("Failed to import environment variables. Are you using an `extbp` image?")
-
-logger = setup_logger(__name__)
-
-
-def get_bypassed_page(url: str) -> Optional[str]:
-    """Fetch HTML content from a URL using an External Cloudflare Resolver.
-
-    Args:
-        url: Target URL
-    Returns:
-        str: HTML content if successful, None otherwise
-    """
-    if not EXT_BYPASSER_URL or not EXT_BYPASSER_PATH:
-        logger.error("Wrong External Bypass configuration. Please check your environment configuration.")
-        return None
-    ext_url = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}"
-    headers = {"Content-Type": "application/json"}
-    data = {
-        "cmd": "request.get",
-        "url": url,
-        "maxTimeout": EXT_BYPASSER_TIMEOUT
-    }
-    response = requests.post(ext_url, headers=headers, json=data)
-    response.raise_for_status()
-    logger.debug(f"External Bypass response for '{url}': {response.json()['status']} - {response.json()['message']}")
-    return response.json()['solution']['response']

compose/docker-compose.lite.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark-lite:
+    image: ghcr.io/calibrain/shelfmark-lite:latest
+    environment:
+      # EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.tor.yml

@@ -0,0 +1,20 @@
+# Routes all traffic through Tor - requires NET_ADMIN capability
+services:
+  shelfmark-tor:
+    image: ghcr.io/calibrain/shelfmark:latest
+    environment:
+      FLASK_PORT: 8084
+      USING_TOR: true
+      PUID: 1000
+      PGID: 1000
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    container_name: shelfmark
+    environment:
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

config.py

@@ -1,101 +0,0 @@
-"""Configuration settings for the book downloader application."""
-
-import os
-from pathlib import Path
-import json
-import env
-from logger import setup_logger
-
-logger = setup_logger(__name__)
-
-for key, value in env.__dict__.items():
-    if not key.startswith('_'):
-        if key == "AA_DONATOR_KEY" and value.strip() != "":
-            value = "REDACTED"
-        logger.info(f"{key}: {value}")
-
-with open("data/book-languages.json") as file:
-    _SUPPORTED_BOOK_LANGUAGE = json.load(file)
-
-# Directory settings
-BASE_DIR = Path(__file__).resolve().parent
-logger.info(f"BASE_DIR: {BASE_DIR}")
-if env.ENABLE_LOGGING:
-    env.LOG_DIR.mkdir(exist_ok=True)
-
-# Create necessary directories
-env.TMP_DIR.mkdir(exist_ok=True)
-env.INGEST_DIR.mkdir(exist_ok=True)
-
-CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
-logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
-logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
-logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
-
-# Network settings
-_custom_dns = env._CUSTOM_DNS.lower().strip()
-_doh_server = ""
-if _custom_dns == "google":
-    CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860:0000:0000:0000:0000:8888", "2001:4860:4860:0000:0000:0000:0000:8844"]
-    _doh_server = "https://dns.google/dns-query"
-elif _custom_dns == "quad9":
-    CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:00fe:0000:0000:0000:0000:0000:00fe", "2620:00fe:0000:0000:0000:0000:0000:0009"]
-    _doh_server = "https://dns.quad9.net/dns-query"
-elif _custom_dns == "cloudflare":
-    CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700:0000:0000:0000:0000:1111", "2606:4700:4700:0000:0000:0000:0000:1001"]
-    _doh_server = "https://cloudflare-dns.com/dns-query"
-elif _custom_dns == "opendns":
-    CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:0119:0035:0000:0000:0000:0000:0035", "2620:0119:0053:0000:0000:0000:0000:0053"]
-    _doh_server = "https://doh.opendns.com/dns-query"
-else:
-    _custom_dns_ip = _custom_dns.split(",")
-    CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()]
-logger.info(f"CUSTOM_DNS: {CUSTOM_DNS}")
-DOH_SERVER = _doh_server
-if env.USE_DOH:
-    DOH_SERVER = _doh_server
-else:
-    DOH_SERVER = ""
-logger.info(f"DOH_SERVER: {DOH_SERVER}")
-
-# Proxy settings
-PROXIES = {}
-if env.HTTP_PROXY:
-    PROXIES["http"] = env.HTTP_PROXY
-if env.HTTPS_PROXY:
-    PROXIES["https"] = env.HTTPS_PROXY
-logger.info(f"PROXIES: {PROXIES}")
-
-# Anna's Archive settings
-AA_BASE_URL = env._AA_BASE_URL
-AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
-AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
-AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
-
-# File format settings
-SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
-logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
-
-# Complex language processing logic kept in config.py
-BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
-BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
-if len(BOOK_LANGUAGE) == 0:
-    BOOK_LANGUAGE = ['en']
-
-# Custom script settings with validation logic
-CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
-if CUSTOM_SCRIPT:
-    if not os.path.exists(CUSTOM_SCRIPT):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
-        CUSTOM_SCRIPT = ""
-    elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
-        CUSTOM_SCRIPT = ""
-
-# Debugging settings
-if not env.USING_EXTERNAL_BYPASSER:
-    # Virtual display settings for debugging internal cloudflare bypasser
-    VIRTUAL_SCREEN_SIZE = (1024, 768)
-    RECORDING_DIR = env.LOG_DIR / "recording"
-    if env.DEBUG:
-        RECORDING_DIR.mkdir(parents=True, exist_ok=True)

docker-compose.bypass-test.yml

@@ -0,0 +1,39 @@
+# Bypass testing - switch between dev build and v1.0.1
+# Usage:
+#   Test dev build:  docker compose -f docker-compose.bypass-test.yml up shelfmark-dev
+#   Test v1.0.1:     docker compose -f docker-compose.bypass-test.yml up shelfmark-stable
+#   Pull latest dev: docker compose -f docker-compose.bypass-test.yml build shelfmark-dev
+#   Pull v1.0.1:     docker compose -f docker-compose.bypass-test.yml pull shelfmark-stable
+
+services:
+  # Dev image from registry
+  shelfmark-dev:
+    image: ghcr.io/calibrain/shelfmark:dev
+    container_name: shelfmark-bypass-dev
+    environment:
+      PUID: 1000
+      PGID: 1000
+      DEBUG: true
+    ports:
+      - 8084:8084
+    volumes:
+      - ./.local/bypass-test/config-dev:/config
+      - ./.local/bypass-test/books:/books
+      - ./.local/bypass-test/log-dev:/var/log/shelfmark
+      - ./.local/bypass-test/tmp:/tmp/shelfmark
+
+  # Stable v1.0.1 for comparison
+  shelfmark-stable:
+    image: ghcr.io/calibrain/shelfmark:1.0.1
+    container_name: shelfmark-bypass-stable
+    environment:
+      PUID: 1000
+      PGID: 1000
+      DEBUG: true
+    ports:
+      - 8085:8084
+    volumes:
+      - ./.local/bypass-test/config-stable:/config
+      - ./.local/bypass-test/books:/books
+      - ./.local/bypass-test/log-stable:/var/log/shelfmark
+      - ./.local/bypass-test/tmp:/tmp/shelfmark

docker-compose.dev.extbp.yml

@@ -0,0 +1,25 @@
+# Local development - External bypasser variant (lite)
+services:
+  shelfmark-lite-dev:
+    extends:
+      file: ./compose/docker-compose.lite.yml
+      service: shelfmark-lite
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark-lite
+    environment:
+      DEBUG: true
+      EXT_BYPASSER_URL: http://flaresolverr:8191
+      EXT_BYPASSER_PATH: /v1
+      EXT_BYPASSER_TIMEOUT: 60000
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads
+
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.dev.tor.yml

@@ -0,0 +1,20 @@
+# Local development - Tor variant
+services:
+  shelfmark-tor-dev:
+    extends:
+      file: ./compose/docker-compose.tor.yml
+      service: shelfmark-tor
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    environment:
+      DEBUG: true
+      USING_TOR: true
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.dev.yml

@@ -1,16 +1,22 @@
+# Local development - builds from source with debug enabled
 services:
-  calibre-web-automated-book-downloader-dev:
+  shelfmark-dev:
     extends:
-      file: ./docker-compose.yml
-      service: calibre-web-automated-book-downloader
+      file: ./compose/docker-compose.yml
+      service: shelfmark
     build:
       context: .
       dockerfile: Dockerfile
-      target: cwa-bd
+      target: shelfmark
+    cap_add:
+      - SYS_PTRACE
     environment:
       DEBUG: true
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
     volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      - ./shelfmark:/app/shelfmark:ro
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.extbp.dev.yml

@@ -1,34 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-extbp-dev:
-    container_name: cwa-bd-extbp-dev
-    extends:
-      file: ./docker-compose.extbp.yml
-      service: calibre-web-automated-book-downloader-extbp
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-extbp
-    environment:
-      DEBUG: true
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
-      USE_CF_BYPASS: true  # Enable Cloudflare bypass (default: true)
-      # External Cloudflare Bypass environment variables
-      EXT_BYPASSER_URL: "http://flaresolverr:8191"  # URL of the external Cloudflare resolver service (used FlareSolverr)
-      EXT_BYPASSER_PATH: "/v1"  # Path for external Cloudflare resolver API (default: /v1)
-      EXT_BYPASSER_TIMEOUT: 60000  # Timeout for external Cloudflare resolver requests (default: 60000)
-    volumes:
-      #- /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      #- /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader
-      - ./deploy/ingest:/cwa-book-ingest
-      - ./deploy/log:/var/log/cwa-book-downloader
-      - ./deploy/tmp:/tmp/cwa-book-downloader
-
-  flaresolverr:  # External Cloudflare resolver service
-    image: ghcr.io/flaresolverr/flaresolverr:v3.3.22
-    container_name: flaresolverr
-    environment:
-      LOG_LEVEL: info
-      LOG_HTML: false
-      CAPTCHA_SOLVER: none
-      TZ: Europe/Rome

docker-compose.extbp.yml

@@ -1,27 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-extbp:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-extbp:latest
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      UID: 1000
-      GID: 100
-      # CWA_DB_PATH: /auth/app.db  # Uncomment to enable authentication
-      # SESSION_COOKIE_SECURE: 'true'  # Set to 'true' if accessing ONLY via HTTPS
-      # DEBUG: 'true'  # Enable debug mode (debug button, verbose logging)
-      EXT_BYPASSER_URL: http://flaresolverr:8191
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above)
-      #- /cwa/config/path/app.db:/auth/app.db:ro
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.test-clients.yml

@@ -0,0 +1,181 @@
+# Test stack for download client development
+# Includes shelfmark + all download clients on same network with shared volumes
+#
+# Usage:
+#   docker compose -f docker-compose.test-clients.yml up -d
+#   # Access shelfmark at http://localhost:8084
+#   # Configure clients in Settings > Prowlarr > Download Clients
+#
+# Web UIs:
+#   - shelfmark:    http://localhost:8084
+#   - Prowlarr:     http://localhost:9696 (no auth by default)
+#   - qBittorrent:  http://localhost:8080 (check container logs for temp password)
+#   - Transmission: http://localhost:9091 (admin / admin)
+#   - Deluge:       http://localhost:8112 (password: deluge)
+#   - NZBGet:       http://localhost:6789 (nzbget / tegbzn6789)
+#   - SABnzbd:      http://localhost:8085 (complete setup wizard for API key)
+#   - rTorrent:     http://localhost:8000 (admin / admin - if auth enabled)
+#
+
+services:
+  shelfmark:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    container_name: test-shelfmark
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      # All client configuration is done via Settings UI
+      # Use Docker service names for URLs:
+      #   - qBittorrent: http://qbittorrent:8080
+      #   - Transmission: http://transmission:9091
+      #   - Deluge Web UI: http://deluge:8112
+      #   - NZBGet: http://nzbget:6789
+      #   - SABnzbd: http://sabnzbd:8080
+      #   - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
+    ports:
+      - "8084:8084"
+    volumes:
+      # Config and state
+      - ./.local/test-clients/shelfmark/config:/config
+      - ./.local/test-clients/shelfmark/log:/var/log/shelfmark
+      # Book destination directory (where completed books go)
+      - ./.local/test-clients/books:/books
+      # Staging directory
+      - ./.local/test-clients/tmp:/tmp/shelfmark
+      # CRITICAL: Mount client download directories so shelfmark can access completed files
+      - ./.local/test-clients/downloads:/downloads
+      # Mount source code for hot-reload (no rebuild needed for Python changes)
+      - ./shelfmark:/app/shelfmark:ro
+      # Mount tests for running pytest in container
+      - ./tests:/app/tests:ro
+      - ./pyproject.toml:/app/pyproject.toml:ro
+      # Mount client configs for integration tests to read credentials
+      - ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
+      - ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
+    depends_on:
+      - nzbget
+      - sabnzbd
+      - qbittorrent
+      - transmission
+      - deluge
+      - rtorrent
+    restart: unless-stopped
+
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:latest
+    container_name: test-prowlarr
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/prowlarr/config:/config
+    ports:
+      - "9696:9696"
+    restart: unless-stopped
+
+  nzbget:
+    image: lscr.io/linuxserver/nzbget:latest
+    container_name: test-nzbget
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/nzbget/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/nzbget/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "6789:6789"  # Web UI / JSON-RPC
+    restart: unless-stopped
+
+  sabnzbd:
+    image: lscr.io/linuxserver/sabnzbd:latest
+    container_name: test-sabnzbd
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/sabnzbd/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8085:8080"  # Web UI (external:internal)
+    restart: unless-stopped
+
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: test-qbittorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - WEBUI_PORT=8080
+    volumes:
+      - ./.local/test-clients/qbittorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/qbittorrent/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "8080:8080"  # Web UI / API
+      - "6882:6881"
+      - "6882:6881/udp"
+    restart: unless-stopped
+
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: test-transmission
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - USER=admin
+      - PASS=admin
+    volumes:
+      - ./.local/test-clients/transmission/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "9091:9091"  # Web UI / RPC
+      - "51413:51413"
+      - "51413:51413/udp"
+    restart: unless-stopped
+
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: test-deluge
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - DELUGE_LOGLEVEL=error
+    volumes:
+      - ./.local/test-clients/deluge/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8112:8112"  # Web UI
+      - "58846:58846"  # Daemon RPC
+      - "6881:6881"
+      - "6881:6881/udp"
+    restart: unless-stopped
+
+  rtorrent:
+    image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
+    container_name: test-rtorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/rtorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8000:8000"  # XMLRPC
+      - "8089:8080"  # ruTorrent Web UI
+      - "9000:9000"  # SCGI port
+      - "50000:50000"  # Incoming connections
+      - "6881:6881/udp"
+    restart: unless-stopped

docker-compose.tor.dev.yml

@@ -1,14 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor-dev:
-    extends:
-      file: ./docker-compose.tor.yml
-      service: calibre-web-automated-book-downloader-tor
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-tor
-    environment:
-      DEBUG: true
-    volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader

docker-compose.tor.yml

@@ -1,26 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      USING_TOR: true
-      # CWA_DB_PATH: /auth/app.db  # Uncomment to enable authentication
-      # SESSION_COOKIE_SECURE: 'true'  # Set to 'true' if accessing ONLY via HTTPS
-      # DEBUG: 'true'  # Enable debug mode (debug button, verbose logging)
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be 
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above)
-      #- /cwa/config/path/app.db:/auth/app.db:ro

docker-compose.yml

@@ -1,32 +0,0 @@
-services:
-  calibre-web-automated-book-downloader:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
-    # Uncomment to build the image from the Dockerfile for local testing changes.
-    # Remember to comment out the image line above.
-    #build: .
-    container_name: calibre-web-automated-book-downloader
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      UID: 1000
-      GID: 100
-      # CWA_DB_PATH: /auth/app.db  # Uncomment to enable authentication (also uncomment volume below)
-      # CALIBRE_WEB_URL: http://localhost:8080 # Uncomment and add your custom library URL to enable "Go To Library" button in the Web UI
-      # SESSION_COOKIE_SECURE: 'true'  # Set to 'true' if accessing ONLY via HTTPS
-      # DEBUG: 'true'  # Enable debug mode (debug button, verbose logging)
-      # Queue management settings
-      MAX_CONCURRENT_DOWNLOADS: 3
-      DOWNLOAD_PROGRESS_UPDATE_INTERVAL: 5
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above)
-      #- /cwa/config/path/app.db:/auth/app.db:ro

docs/configuration.md

@@ -0,0 +1,125 @@
+# Directory and Volume Setup
+
+This guide explains how to configure directories and Docker volumes for Shelfmark. It focuses on the difference between the destination folder and your download client paths, and how to make those paths line up inside containers.
+
+## Conceptual Overview
+
+```
+DIRECT DOWNLOADS
+
+Shelfmark downloads directly -> destination
+
+TORRENT / USENET
+
+Prowlarr -> Download client saves to <client path>
+         -> Shelfmark reads from <client path>
+         -> Shelfmark processes to destination
+```
+
+Key point: For torrent and usenet downloads, Shelfmark must see the same file path that your download client reports. The container path must match in both containers.
+
+## Direct Download Setup
+
+Direct downloads do not use an external download client. A simple two-folder setup is enough.
+
+Required volumes:
+
+| Container path | Purpose | Notes |
+| --- | --- | --- |
+| `/config` | Settings, database, cover cache | Configurable via `CONFIG_DIR` |
+| `/books` | Destination folder for completed files | Configurable via `INGEST_DIR` and Settings -> Downloads -> Destination |
+
+Example `docker-compose`:
+
+```yaml
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    volumes:
+      - /path/to/config:/config
+      - /path/to/books:/books
+```
+
+Notes:
+- Point `/books` to your library ingest folder (Calibre-Web, Booklore, Audiobookshelf, etc) for automatic import.
+- If you set Books Output Mode to Booklore (API), books are uploaded via API instead of written to `/books`. Audiobooks still use a destination folder.
+- Ensure `PUID`/`PGID` (or legacy `UID`/`GID`) match the owner of the host directories to avoid permission errors.
+
+## Torrent / Usenet Setup
+
+For torrents and usenet, your download client reports a path (for example `/data/torrents/books/MyBook.epub`). Shelfmark must be able to read that exact path inside its own container.
+
+Required volumes:
+
+| Container path | Purpose | Notes |
+| --- | --- | --- |
+| `/config` | Settings, database, cover cache | Configurable via `CONFIG_DIR` |
+| `/books` | Destination folder for processed files | Configurable via `INGEST_DIR` |
+| `<client path>` | Download client path | Must match the download client container path exactly |
+
+Side-by-side example with qBittorrent:
+
+```yaml
+services:
+  shelfmark:
+    volumes:
+      - /path/to/config:/config
+      - /path/to/books:/books
+      - /path/to/downloads:/data/torrents  # Must match client
+
+  qbittorrent:
+    volumes:
+      - /path/to/downloads:/data/torrents  # Same container path
+```
+
+Host paths can be anything. The container path (for example `/data/torrents`) must be identical in both containers.
+
+### Remote Path Mappings
+
+If paths cannot match (different machines or a fixed setup), use Remote Path Mappings.
+
+Where to configure:
+- Settings -> Advanced -> Remote Path Mappings
+
+Example:
+- Client reports `/data/torrents/books/...`
+- Shelfmark can see the same files at `/downloads/books/...`
+- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
+
+## File Processing Options
+
+### Transfer Method (Torrent / Usenet Only)
+
+Available methods:
+- Copy (default). Works everywhere.
+- Hardlink. Preserves seeding without duplicating files.
+
+Hardlink requirements and behavior:
+- Source and destination must be on the same filesystem.
+- If hardlinking is enabled but not possible, Shelfmark falls back to copying.
+- Archive extraction is disabled while hardlinking is enabled.
+- Do not use hardlinking if your destination is a library ingest folder.
+
+### File Organization
+
+Shelfmark supports three organization modes for the destination:
+- None. Keep original filenames from the source.
+- Rename Only. Rename files using a template.
+- Rename and Organize. Create folders and rename using templates. Do not use with ingest folders.
+
+Configure templates in Settings -> Downloads. Template syntax details are documented separately.
+
+## Common Mistakes
+
+- "Download failed - file not found": Path mismatch between Shelfmark and the download client. Ensure container paths match or use Remote Path Mappings.
+- "Permission denied": `PUID`/`PGID` do not match the host directories. Ensure Shelfmark can read the client path and write to the destination.
+- "Hardlinks not working" or "Files being copied instead": Source and destination are on different filesystems. Move the destination or accept copy fallback.
+- "Downloads work but library does not see them": Destination does not point to the library ingest folder. Check Settings -> Downloads -> Destination.
+- CIFS/SMB shares: Use the `nobrl` mount option to avoid database lock errors. Example: `//server/share /mnt/share cifs nobrl,... 0 0`
+
+## Related Documentation
+
+- Environment Variables Reference: `docs/environment-variables.md`
+- Custom Scripts: `docs/custom-scripts.md`
+- Installation: `docs/installation.md`
+- Troubleshooting: `docs/troubleshooting.md`

docs/custom-scripts.md

@@ -0,0 +1,184 @@
+# Custom Scripts
+
+Shelfmark can run an executable you provide after a download task completes successfully. The script runs after the selected output has finished (for example: transfer to the folder destination, or upload to Booklore).
+
+
+## Quick Start (Recommended)
+
+1. Put your script on the machine that runs Shelfmark.
+1. Make it executable.
+1. Set it in Shelfmark (Settings -> Advanced -> Custom Script Path).
+
+Example:
+
+```bash
+chmod +x /path/to/your/scripts/post_process.sh
+```
+
+### Docker Users
+
+If you run Shelfmark in Docker, the script must exist inside the container. The easiest way is to mount a folder of scripts, then point Shelfmark at the container path in the UI.
+
+```yaml
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    volumes:
+      - /path/to/your/scripts:/scripts:ro
+```
+
+Then set:
+
+- Settings -> Advanced -> Custom Script Path: `/scripts/post_process.sh`
+
+<details>
+<summary>Docker Compose: Configure Via Environment Variables (Optional)</summary>
+
+```yaml
+services:
+  shelfmark:
+    environment:
+      - CUSTOM_SCRIPT=/scripts/post_process.sh
+      - CUSTOM_SCRIPT_PATH_MODE=absolute
+      - CUSTOM_SCRIPT_JSON_PAYLOAD=true
+```
+
+</details>
+
+## Script Behaviour
+
+When enabled, Shelfmark runs your script once per successful task:
+
+```bash
+<custom_script_path> "<target_path>"
+```
+
+- `$1` is always set to the target path.
+- If **Custom Script JSON Payload** is enabled, Shelfmark writes a JSON document to stdin (UTF-8).
+- If JSON payload is disabled, stdin is empty (EOF).
+- Timeout: 300 seconds (5 minutes)
+- Exit code: `0` = success; anything else = the task is marked as **Error**
+- Concurrency: downloads can run in parallel, so your script may be invoked concurrently for different tasks.
+- Runtime: the script runs inside the Shelfmark container (if you use Docker) under the same user as Shelfmark.
+
+## The Target Path (`$1`)
+
+Shelfmark chooses a "best single path" for the task:
+
+- If the output produced exactly one local file: that file path.
+- If the output produced multiple local files: a directory path (the common parent directory of those files).
+
+What the target path refers to depends on the output mode:
+
+- Folder output (`output.mode=folder`, `phase=post_transfer`): the final imported file or folder inside your destination.
+- Booklore output (`output.mode=booklore`, `phase=post_upload`): the local file or folder that was uploaded (the destination is remote).
+
+By default, `$1` is an absolute path inside the Shelfmark container (or on your host, if you are not using Docker).
+
+## JSON Payload (stdin)
+
+Configure in: Settings -> Advanced -> Custom Script JSON Payload
+
+When enabled, Shelfmark sends a versioned JSON payload to your script via stdin (and still passes `$1`). This is the recommended way to write robust scripts, especially for multi-file imports (audiobooks) and output-specific context (like Booklore).
+
+- The JSON payload always includes absolute paths in `paths.*`, even if you set Custom Script Path Mode to `relative` for `$1`.
+- `output.mode` tells you which output ran.
+- `output.details` is output-specific. For Booklore output, `output.details.booklore` includes connection details such as `base_url`, `library_id`, and `path_id`.
+- `phase` indicates when the script is running. Current values: `post_transfer` (folder output), `post_upload` (Booklore output).
+- `transfer` is only included for outputs that do a local transfer (for example the folder output).
+
+If JSON payload is disabled, stdin is empty (EOF). Don't `cat` stdin unless you've enabled the payload.
+
+Example payload shape:
+
+```json
+{
+  "version": 1,
+  "phase": "post_transfer",
+  "task": {
+    "task_id": "abc123",
+    "source": "direct",
+    "title": "Foundation",
+    "author": "Isaac Asimov"
+  },
+  "output": {
+    "mode": "folder",
+    "organization_mode": "organize"
+  },
+  "paths": {
+    "destination": "/data/library/books",
+    "target": "/data/library/books/Isaac Asimov/Foundation/Foundation.epub",
+    "final_paths": [
+      "/data/library/books/Isaac Asimov/Foundation/Foundation.epub"
+    ]
+  },
+  "transfer": {
+    "op_counts": {"copy": 1, "move": 0, "hardlink": 0},
+    "use_hardlink": false,
+    "is_torrent": false,
+    "preserve_source": false
+  }
+}
+```
+
+Example (bash + jq) (JSON payload must be enabled):
+
+```bash
+payload="$(cat)"
+mode="$(echo "$payload" | jq -r '.output.mode')"
+title="$(echo "$payload" | jq -r '.task.title')"
+final_paths="$(echo "$payload" | jq -r '.paths.final_paths[]')"
+echo "mode=$mode title=$title" >&2
+echo "$final_paths" >&2
+```
+
+Example (Python) (works whether JSON payload is enabled or not):
+
+```python
+#!/usr/bin/env python3
+import json
+import sys
+
+target = sys.argv[1]
+raw = sys.stdin.read()
+payload = json.loads(raw) if raw.strip() else None
+
+print(f"target={target}", file=sys.stderr)
+if payload:
+    print(f"mode={payload['output']['mode']} phase={payload['phase']}", file=sys.stderr)
+```
+
+<details>
+<summary>Advanced Options</summary>
+
+### Absolute vs Relative Target Paths
+
+Configure in: Settings -> Advanced -> Custom Script Path Mode
+
+This setting controls what gets passed as `$1`:
+
+- `absolute` (default): pass an absolute path.
+- `relative`: pass a path relative to the output's "destination root", and run the script with `$PWD` set to that root.
+
+For folder output, the destination root is your configured destination folder. For Booklore output, it's the local upload folder.
+
+Example (folder destination is `/data/library/books`, and the imported file ended up in `Isaac Asimov/Foundation/Foundation.epub`):
+
+```bash
+# Absolute mode:
+$PWD is unchanged
+$1 = /data/library/books/Isaac Asimov/Foundation/Foundation.epub
+
+# Relative mode:
+$PWD = /data/library/books
+$1 = Isaac Asimov/Foundation/Foundation.epub
+```
+
+Note: if the target is the destination folder itself, `relative` mode may pass `.`.
+
+</details>
+
+## Notes And Caveats
+
+- **Hardlinks and torrents:** if you use hardlinking to keep seeding, avoid scripts that modify file contents, since hardlinked files share data with the seeding copy.
+- **Booklore output mode:** scripts run after upload. `$1` will point at the local uploaded file (or staging folder).

docs/dev/index.md

@@ -0,0 +1,3 @@
+# Developer Documentation
+
+TODO

docs/dev/plugin-settings.md

@@ -0,0 +1,654 @@
+# Plugin Settings Integration Guide
+
+This guide explains how to add configuration settings to plugins (Metadata Providers and Release Sources) so they appear in the Settings UI.
+
+## Overview
+
+The settings system uses a decorator-based registration pattern. Plugins register their settings when their module is imported, and the frontend dynamically renders the appropriate UI based on the schema provided by the backend.
+
+**Key features:**
+- Settings are defined in Python and automatically rendered in the React frontend
+- Values persist across container restarts via JSON config files
+- Changes take effect immediately without restart (unless marked otherwise)
+
+## Quick Start
+
+Add settings to your plugin in 3 steps:
+
+```python
+from shelfmark.core.settings_registry import (
+    register_settings,
+    TextField,
+    PasswordField,
+    ActionButton,
+)
+
+@register_settings(
+    name="my_plugin",           # Unique identifier
+    display_name="My Plugin",   # Shown in sidebar
+    icon="wrench",              # Icon name
+    order=100,                  # Sort order (lower = higher in list)
+    group="metadata_providers"  # Optional: group in sidebar
+)
+def my_plugin_settings():
+    return [
+        PasswordField(
+            key="MY_PLUGIN_API_KEY",
+            label="API Key",
+            description="Your API key from the provider",
+            required=True,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+def _test_connection():
+    # Perform connection test
+    return {"success": True, "message": "Connected successfully!"}
+```
+
+## Available Field Types
+
+### TextField
+
+Single-line text input for strings.
+
+```python
+TextField(
+    key="MY_SETTING",           # Config key
+    label="Setting Name",       # Display label
+    description="Help text",    # Optional description below field
+    default="",                 # Default value
+    placeholder="Enter value",  # Placeholder text
+    max_length=100,             # Optional max characters
+    required=False,             # Is this field required?
+    requires_restart=False,     # Does changing this need a restart?
+    show_when=None,             # Conditional visibility (see below)
+    disabled_when=None,         # Conditional disable (see below)
+)
+```
+
+### PasswordField
+
+Masked input for sensitive values (API keys, passwords). Values are never echoed back to the frontend.
+
+```python
+PasswordField(
+    key="API_KEY",
+    label="API Key",
+    description="Your secret API key",
+    placeholder="sk-...",
+    required=True,
+)
+```
+
+### NumberField
+
+Numeric input with optional min/max constraints.
+
+```python
+NumberField(
+    key="TIMEOUT",
+    label="Timeout (seconds)",
+    description="Connection timeout in seconds",
+    default=30,
+    min_value=5,
+    max_value=300,
+    step=1,                     # Increment step
+    required=False,
+)
+```
+
+### CheckboxField
+
+Toggle switch for boolean values.
+
+```python
+CheckboxField(
+    key="ENABLE_FEATURE",
+    label="Enable Feature",
+    description="Turn this feature on or off",
+    default=False,
+)
+```
+
+### SelectField
+
+Dropdown for single-choice selection.
+
+```python
+SelectField(
+    key="LOG_LEVEL",
+    label="Log Level",
+    description="Logging verbosity",
+    default="info",
+    options=[
+        {"value": "debug", "label": "Debug"},
+        {"value": "info", "label": "Info"},
+        {"value": "warning", "label": "Warning"},
+        {"value": "error", "label": "Error"},
+    ],
+)
+```
+
+### MultiSelectField
+
+Multi-choice selection from a list of options.
+
+```python
+MultiSelectField(
+    key="SUPPORTED_FORMATS",
+    label="Supported Formats",
+    description="Select which formats to support",
+    default=["epub", "mobi"],
+    options=[
+        {"value": "epub", "label": "EPUB"},
+        {"value": "mobi", "label": "MOBI"},
+        {"value": "pdf", "label": "PDF"},
+        {"value": "azw3", "label": "AZW3"},
+    ],
+)
+```
+
+### ActionButton
+
+Button that executes a callback function. Does not store a value.
+
+```python
+ActionButton(
+    key="test_connection",      # Unique key for the action
+    label="Test Connection",    # Button text
+    description="Test the API connection",
+    style="primary",            # "default", "primary", or "danger"
+    callback=my_callback_fn,    # Function to execute
+)
+
+def my_callback_fn():
+    """Callback must return dict with 'success' and 'message' keys."""
+    try:
+        # Perform action
+        return {"success": True, "message": "Connection successful!"}
+    except Exception as e:
+        return {"success": False, "message": f"Failed: {str(e)}"}
+```
+
+### HeadingField
+
+Display-only section heading with optional link. Does not store a value.
+
+```python
+HeadingField(
+    key="section_heading",      # Unique key
+    title="Configuration",      # Heading text
+    description="Configure the plugin settings below",
+    link_url="https://example.com/docs",  # Optional link
+    link_text="View Documentation",       # Link text
+)
+```
+
+### CustomComponentField
+
+Render a frontend-registered custom settings component while still using the
+decorator-based schema.
+
+```python
+from shelfmark.core.settings_registry import CustomComponentField
+
+CustomComponentField(
+    key="request_policy_editor",
+    component="request_policy_grid",  # frontend registry key
+    label="Request Policy Rules",
+    description="Custom editor for policy defaults and matrix rules.",
+    value_fields=[
+        SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
+        SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
+        TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
+    ],
+    wrap_in_field_wrapper=True,  # use standard FieldWrapper label/description layout
+)
+```
+
+When `value_fields` is provided, those backing fields are included in
+serialization/save/validation automatically and are hidden from the default renderer.
+
+## Common Field Properties
+
+All field types support these common properties:
+
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `key` | `str` | Required | Unique identifier for this setting |
+| `label` | `str` | Required | Display label in the UI |
+| `description` | `str` | `""` | Help text shown below the field |
+| `default` | `Any` | `None` | Default value if not set |
+| `required` | `bool` | `False` | Whether the field must have a value |
+| `disabled` | `bool` | `False` | Disable the field (greyed out) |
+| `disabled_reason` | `str` | `""` | Explanation shown when disabled |
+| `requires_restart` | `bool` | `False` | Whether changes require container restart |
+| `show_when` | `dict` | `None` | Conditional visibility (see below) |
+| `disabled_when` | `dict` | `None` | Conditional disable (see below) |
+| `hidden_in_ui` | `bool` | `False` | Hide from default renderer but keep in schema/save path |
+
+## Conditional Visibility
+
+Fields can be shown/hidden based on other field values using `show_when`:
+
+```python
+# Only show DNS servers field when custom DNS is selected
+TextField(
+    key="CUSTOM_DNS_SERVERS",
+    label="DNS Servers",
+    description="Comma-separated DNS server IPs",
+    show_when={"field": "DNS_PROVIDER", "value": "manual"},
+)
+```
+
+The field will only be visible when the referenced field has the specified value.
+
+## Conditional Disable
+
+Fields can be enabled/disabled based on other field values using `disabled_when`:
+
+```python
+# Disable timeout field when feature is disabled
+NumberField(
+    key="FEATURE_TIMEOUT",
+    label="Timeout (seconds)",
+    description="Request timeout",
+    default=30,
+    disabled_when={
+        "field": "FEATURE_ENABLED",
+        "value": False,
+        "reason": "Enable the feature first"
+    },
+)
+```
+
+The field will be greyed out with the specified reason when the condition is met.
+
+## Settings Groups
+
+Register a group to organize related settings tabs in the sidebar:
+
+```python
+from shelfmark.core.settings_registry import register_group
+
+# Register a group (do this once, usually in a central config file)
+register_group(
+    name="my_group",
+    display_name="My Group",
+    icon="folder",
+    order=50,
+)
+
+# Then register settings to the group
+@register_settings(
+    name="plugin_a",
+    display_name="Plugin A",
+    icon="puzzle",
+    order=51,
+    group="my_group",  # Assigns to the group
+)
+def plugin_a_settings():
+    return [...]
+```
+
+**Existing groups:**
+- `direct_download` (order=20): For download-related settings
+- `metadata_providers` (order=50): For metadata provider plugins
+
+## Value Resolution Priority
+
+Settings values are resolved in this order (highest priority first):
+
+1. **Config File** - Stored in `CONFIG_DIR/plugins/<tab_name>.json`
+2. **Field Default** - Value specified in the field definition
+
+The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdirectory.
+
+## Reading Setting Values
+
+Use the `config` singleton to read setting values in your plugin code:
+
+```python
+from shelfmark.core.config import config
+
+# Get a setting value with default fallback
+api_key = config.get("MY_PLUGIN_API_KEY", "")
+timeout = config.get("MY_PLUGIN_TIMEOUT", 30)
+
+# Or access as attributes (raises AttributeError if not found)
+api_key = config.MY_PLUGIN_API_KEY
+
+# Check all cached settings
+all_settings = config.get_all()
+```
+
+The config singleton:
+- Automatically resolves values from config files with field defaults as fallback
+- Caches values for performance
+- Refreshes automatically when settings are updated via the UI
+
+## Complete Example: Metadata Provider
+
+Here's a complete example for a metadata provider plugin:
+
+```python
+# shelfmark/metadata_providers/my_provider.py
+
+from shelfmark.metadata_providers.base import (
+    MetadataProvider,
+    register_provider,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_connection():
+    """Test API connection callback."""
+    api_key = config.get("MY_PROVIDER_API_KEY", "")
+    if not api_key:
+        return {"success": False, "message": "API key not configured"}
+
+    try:
+        # Perform actual connection test
+        # response = requests.get(...)
+        return {"success": True, "message": "Connected to My Provider API"}
+    except Exception as e:
+        return {"success": False, "message": f"Connection failed: {str(e)}"}
+
+
+@register_settings(
+    name="my_provider",
+    display_name="My Provider",
+    icon="book",
+    order=53,
+    group="metadata_providers",
+)
+def my_provider_settings():
+    """Define settings for this metadata provider."""
+    return [
+        HeadingField(
+            key="my_provider_heading",
+            title="My Provider",
+            description="A metadata provider for book information",
+            link_url="https://myprovider.com",
+            link_text="Visit My Provider",
+        ),
+        PasswordField(
+            key="MY_PROVIDER_API_KEY",
+            label="API Key",
+            description="Your My Provider API key",
+            placeholder="Enter your API key",
+            required=True,
+        ),
+        CheckboxField(
+            key="MY_PROVIDER_INCLUDE_COVERS",
+            label="Include Cover Images",
+            description="Fetch cover images when searching",
+            default=True,
+        ),
+        TextField(
+            key="MY_PROVIDER_BASE_URL",
+            label="API Base URL",
+            description="Override the default API endpoint",
+            default="https://api.myprovider.com/v1",
+            required=False,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            description="Verify your API key works",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+
+@register_provider("my_provider")
+class MyProvider(MetadataProvider):
+    """My Provider metadata implementation."""
+
+    name = "my_provider"
+    display_name = "My Provider"
+    requires_auth = True
+
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
+        self.base_url = config.get(
+            "MY_PROVIDER_BASE_URL",
+            "https://api.myprovider.com/v1"
+        )
+
+    def is_available(self) -> bool:
+        return bool(self.api_key)
+
+    def search(self, query: str):
+        # Implementation...
+        pass
+
+    def get_book(self, book_id: str):
+        # Implementation...
+        pass
+```
+
+## Complete Example: Release Source
+
+Here's a complete example for a release source plugin:
+
+```python
+# shelfmark/release_sources/my_source.py
+
+from shelfmark.release_sources.base import (
+    ReleaseSource,
+    DownloadHandler,
+    register_source,
+    register_handler,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    NumberField,
+    CheckboxField,
+    SelectField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_source():
+    """Test source availability callback."""
+    base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+    try:
+        # Test connectivity
+        return {"success": True, "message": f"Source available at {base_url}"}
+    except Exception as e:
+        return {"success": False, "message": f"Source unavailable: {str(e)}"}
+
+
+@register_settings(
+    name="my_source",
+    display_name="My Source",
+    icon="download",
+    order=25,
+    group="direct_download",
+)
+def my_source_settings():
+    """Define settings for this release source."""
+    return [
+        HeadingField(
+            key="my_source_heading",
+            title="My Source Configuration",
+            description="Configure the My Source download provider",
+        ),
+        CheckboxField(
+            key="MY_SOURCE_ENABLED",
+            label="Enable My Source",
+            description="Include My Source in download fallback chain",
+            default=True,
+        ),
+        TextField(
+            key="MY_SOURCE_URL",
+            label="Source URL",
+            description="Base URL for the source",
+            default="https://mysource.com",
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        NumberField(
+            key="MY_SOURCE_TIMEOUT",
+            label="Timeout (seconds)",
+            description="Request timeout",
+            default=30,
+            min_value=10,
+            max_value=120,
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        SelectField(
+            key="MY_SOURCE_PRIORITY",
+            label="Priority",
+            description="Where in the fallback chain to try this source",
+            default="normal",
+            options=[
+                {"value": "high", "label": "High (try first)"},
+                {"value": "normal", "label": "Normal"},
+                {"value": "low", "label": "Low (try last)"},
+            ],
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        ActionButton(
+            key="test_source",
+            label="Test Source",
+            description="Check if the source is accessible",
+            style="primary",
+            callback=_test_source,
+        ),
+    ]
+
+
+@register_source("my_source")
+class MySource(ReleaseSource):
+    """My Source release source implementation."""
+
+    name = "my_source"
+    display_name = "My Source"
+
+    def __init__(self):
+        self.enabled = config.get("MY_SOURCE_ENABLED", True)
+        self.base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+        self.timeout = config.get("MY_SOURCE_TIMEOUT", 30)
+
+    def is_available(self) -> bool:
+        return self.enabled
+
+    def search(self, book):
+        # Implementation...
+        pass
+
+
+@register_handler("my_source")
+class MySourceHandler(DownloadHandler):
+    """Handler for downloading from My Source."""
+
+    name = "my_source"
+
+    def download(self, release, output_path):
+        # Implementation...
+        pass
+```
+
+## Best Practices
+
+1. **Use descriptive keys**: Keys should be uppercase and prefixed with your plugin name (e.g., `MY_PLUGIN_API_KEY`)
+
+2. **Provide helpful descriptions**: Include enough detail in descriptions to help users understand what each setting does
+
+3. **Set sensible defaults**: Users should be able to get started without configuring everything
+
+4. **Use conditional visibility**: Hide advanced options behind enabling checkboxes to reduce UI clutter
+
+5. **Include a test button**: ActionButtons that test connections help users verify their configuration
+
+6. **Mark restart-required settings**: Use `requires_restart=True` for settings that can't be applied live
+
+7. **Group related settings**: Use HeadingField to visually separate sections, and put plugins in appropriate groups
+
+8. **Handle missing values gracefully**: Always provide fallbacks when reading settings in your code
+
+## API Reference
+
+### Backend Routes
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/api/settings` | Get all settings tabs, groups, and values |
+| GET | `/api/settings/<tab_name>` | Get a specific settings tab |
+| PUT | `/api/settings/<tab_name>` | Update settings for a tab |
+| POST | `/api/settings/<tab_name>/action/<action_key>` | Execute an action button callback |
+
+### Response Format
+
+**GET /api/settings**
+```json
+{
+  "groups": [
+    {"name": "direct_download", "displayName": "Direct Download", "icon": "download", "order": 20}
+  ],
+  "tabs": [
+    {
+      "name": "my_plugin",
+      "displayName": "My Plugin",
+      "icon": "book",
+      "order": 53,
+      "group": "metadata_providers",
+      "fields": [
+        {
+          "type": "password",
+          "key": "MY_PLUGIN_API_KEY",
+          "label": "API Key",
+          "description": "Your API key",
+          "hasValue": true,
+          "value": "",
+          "required": true,
+          "disabled": false,
+          "requiresRestart": false
+        }
+      ]
+    }
+  ]
+}
+```
+
+**PUT /api/settings/<tab_name>**
+```json
+// Request
+{"MY_PLUGIN_API_KEY": "new-value", "MY_PLUGIN_TIMEOUT": 60}
+
+// Response
+{
+  "success": true,
+  "message": "Settings updated",
+  "updated": ["MY_PLUGIN_API_KEY", "MY_PLUGIN_TIMEOUT"],
+  "requiresRestart": false
+}
+```
+
+**POST /api/settings/<tab_name>/action/<action_key>**
+```json
+// Response
+{
+  "success": true,
+  "message": "Connection successful!"
+}
+```

docs/index.md

@@ -0,0 +1,3 @@
+# Shelfmark Documentation
+
+TODO

docs/installation.md

@@ -0,0 +1,3 @@
+# Installation
+
+TODO

docs/reverse-proxy.md

@@ -0,0 +1,147 @@
+# Reverse Proxy & Subpath Hosting
+
+Shelfmark can run behind a reverse proxy at the root path (recommended) or under a subpath like `/shelfmark`.
+
+## Root path setup (Recommended)
+
+If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`), leave `URL_BASE` empty. This is the simplest option and avoids extra subpath configuration.
+
+```nginx
+server {
+    listen 443 ssl;
+    server_name shelfmark.example.com;
+
+    location / {
+        proxy_pass http://shelfmark:8084;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
+```
+
+## Subpath setup
+
+Running Shelfmark under a subpath like `/shelfmark` is supported without extra rewrite rules.
+
+### 1. Set the base path in Shelfmark
+
+- **UI**: Settings → Advanced → Base Path → `/shelfmark/`
+- **Environment variable**: `URL_BASE=/shelfmark/`
+
+### 2. Configure your reverse proxy
+
+All Shelfmark paths (UI, API, assets, Socket.IO) are served under the base path. A single location block is enough.
+
+---
+
+### Without Authentication Proxy
+
+**Complete Nginx configuration for subpath deployment:**
+
+```nginx
+location /shelfmark/ {
+    proxy_pass http://shelfmark:8084/shelfmark/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+    proxy_send_timeout 86400;
+    proxy_buffering off;
+}
+```
+
+---
+
+### With Authentication Proxy (Authelia, Authentik, etc.)
+
+Shelfmark supports Proxy Authentication. When enabled, Shelfmark trusts the authenticated user from headers set by your auth proxy.
+
+#### Shelfmark Settings
+
+Configure in Settings → Security:
+
+| Setting | Value |
+|---------|-------|
+| Authentication Method | Proxy Authentication |
+| Proxy Auth User Header | `Remote-User` |
+| Proxy Auth Logout URL | `https://auth.example.com/logout` |
+| Proxy Auth Admin Group Header | `Remote-Groups` |
+| Proxy Auth Admin Group Name | `admins` (or your admin group) |
+
+#### Nginx Configuration with Authelia
+
+This example uses Authelia snippets. Adapt for your auth proxy.
+
+**Authelia auth request snippet** (`/etc/nginx/snippets/authelia-authrequest.conf`):
+
+```nginx
+location /authelia {
+    internal;
+    proxy_pass http://authelia:9091/api/authz/auth-request;
+    proxy_pass_request_body off;
+    proxy_set_header Content-Length "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
+    proxy_set_header X-Original-Method $request_method;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+}
+```
+
+**Authelia location snippet** (`/etc/nginx/snippets/authelia-location.conf`):
+
+```nginx
+auth_request /authelia;
+auth_request_set $target_url $scheme://$http_host$request_uri;
+auth_request_set $user $upstream_http_remote_user;
+auth_request_set $groups $upstream_http_remote_groups;
+auth_request_set $name $upstream_http_remote_name;
+auth_request_set $email $upstream_http_remote_email;
+proxy_set_header Remote-User $user;
+proxy_set_header Remote-Groups $groups;
+proxy_set_header Remote-Name $name;
+proxy_set_header Remote-Email $email;
+error_page 401 =302 https://auth.example.com/?rd=$target_url;
+```
+
+**Complete Nginx configuration with Authelia:**
+
+```nginx
+# Include Authelia auth endpoint in your server block
+include /etc/nginx/snippets/authelia-authrequest.conf;
+
+# Main shelfmark location
+location /shelfmark/ {
+    include /etc/nginx/snippets/authelia-location.conf;
+
+    proxy_pass http://shelfmark:8084/shelfmark/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+    proxy_send_timeout 86400;
+    proxy_buffering off;
+}
+```
+
+---
+
+## Health checks
+
+Health checks work at `/shelfmark/api/health` when using a subpath configuration.

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+# Troubleshooting
+
+TODO

docs/url-search-parameters.md

@@ -0,0 +1,75 @@
+# URL Search Parameters
+
+You can trigger searches directly via URL by adding query parameters. This enables bookmarking searches and sharing links.
+
+## Basic Usage
+
+```
+http://your-server:8084/?q=harry+potter
+```
+
+## Supported Parameters
+
+| Parameter | Description | Example |
+|-----------|-------------|---------|
+| `q` or `query` | Main search query | `/?q=dune` |
+| `author` | Filter by author name | `/?author=frank+herbert` |
+| `title` | Filter by book title | `/?title=foundation` |
+| `isbn` | Filter by ISBN | `/?isbn=978-0747532699` |
+| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
+| `format` | Filter by file format | `/?format=epub` |
+| `content` | Filter by content type | `/?content=fiction` |
+| `sort` | Sort order for results | `/?sort=newest` |
+
+## Multiple Values
+
+Some parameters support multiple values by repeating the parameter:
+
+```
+/?lang=en&lang=de&lang=fr
+/?format=epub&format=mobi&format=azw3
+```
+
+## Examples
+
+**Simple search:**
+```
+/?q=lord+of+the+rings
+```
+
+**Search with author filter:**
+```
+/?q=dune&author=frank+herbert
+```
+
+**Search with format and language:**
+```
+/?q=harry+potter&format=epub&lang=en
+```
+
+**Author search with multiple formats:**
+```
+/?author=stephen+king&format=epub&format=mobi
+```
+
+**Search with sort order:**
+```
+/?q=science+fiction&sort=newest
+```
+
+## Search Mode Behavior
+
+### Direct Download Mode (default)
+
+All parameters are used to filter results from the direct download source.
+
+### Universal Mode
+
+Only `q` and `sort` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
+
+## Notes
+
+- URL parameters are read once on page load
+- The URL is not updated when you perform searches manually
+- Spaces should be encoded as `+` or `%20`
+- Invalid or unknown parameters are silently ignored

downloader.py

@@ -1,138 +0,0 @@
-"""Network operations manager for the book downloader application."""
-
-import network
-network.init()
-import requests
-import time
-from io import BytesIO
-from typing import Optional
-from urllib.parse import urlparse
-from tqdm import tqdm
-from typing import Callable
-from threading import Event
-from logger import setup_logger
-from config import PROXIES
-from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER
-if USE_CF_BYPASS:
-    if USING_EXTERNAL_BYPASSER:
-        from cloudflare_bypasser_external import get_bypassed_page
-    else:
-        from cloudflare_bypasser import get_bypassed_page
-
-logger = setup_logger(__name__)
-
-
-def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False, status_callback: Optional[Callable[[str], None]] = None) -> str:
-    """Fetch HTML content from a URL with retry mechanism.
-    
-    Args:
-        url: Target URL
-        retry: Number of retry attempts
-        use_bypasser: Whether to use Cloudflare bypasser
-        status_callback: Optional callback for status updates
-        
-    Returns:
-        str: HTML content if successful, None otherwise
-    """
-    response = None
-    try:
-        logger.debug(f"html_get_page: {url}, retry: {retry}, use_bypasser: {use_bypasser}")
-        if use_bypasser and USE_CF_BYPASS:
-            if status_callback:
-                status_callback("bypassing")
-            logger.info(f"GET Using Cloudflare Bypasser for: {url}")
-            return get_bypassed_page(url)
-        else:
-            logger.info(f"GET: {url}")
-            response = requests.get(url, proxies=PROXIES)
-            response.raise_for_status()
-            logger.debug(f"Success getting: {url}")
-            time.sleep(1)
-        return str(response.text)
-        
-    except Exception as e:
-        if retry == 0:
-            logger.error_trace(f"Failed to fetch page: {url}, error: {e}")
-            return ""
-        
-        if use_bypasser and USE_CF_BYPASS:
-            logger.warning(f"Exception while using cloudflare bypass for URL: {url}")
-            logger.warning(f"Exception: {e}")
-            logger.warning(f"Response: {response}")
-        elif response is not None and response.status_code == 404:
-            logger.warning(f"404 error for URL: {url}")
-            return ""
-        elif response is not None and response.status_code == 403:
-            logger.warning(f"403 detected for URL: {url}. Should retry using cloudflare bypass.")
-            return html_get_page(url, retry - 1, True, status_callback)
-            
-        sleep_time = DEFAULT_SLEEP * (MAX_RETRY - retry + 1)
-        logger.warning(
-            f"Retrying GET {url} in {sleep_time} seconds due to error: {e}"
-        )
-        time.sleep(sleep_time)
-        return html_get_page(url, retry - 1, use_bypasser, status_callback)
-
-def download_url(link: str, size: str = "", progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None) -> Optional[BytesIO]:
-    """Download content from URL into a BytesIO buffer.
-    
-    Args:
-        link: URL to download from
-        
-    Returns:
-        BytesIO: Buffer containing downloaded content if successful
-    """
-    try:
-        logger.info(f"Downloading from: {link}")
-        response = requests.get(link, stream=True, proxies=PROXIES)
-        response.raise_for_status()
-
-        total_size : float = 0.0
-        try:
-            # we assume size is in MB
-            total_size = float(size.strip().replace(" ", "").replace(",", ".").upper()[:-2].strip()) * 1024 * 1024
-        except:
-            total_size = float(response.headers.get('content-length', 0))
-        
-        buffer = BytesIO()
-
-        # Initialize the progress bar with your guess
-        pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
-        for chunk in response.iter_content(chunk_size=1000):
-            buffer.write(chunk)
-            pbar.update(len(chunk))
-            if progress_callback is not None:
-                progress_callback(pbar.n * 100.0 / total_size)
-            if cancel_flag is not None and cancel_flag.is_set():
-                logger.info(f"Download cancelled: {link}")
-                return None
-            
-        pbar.close()
-        if buffer.tell() * 0.1 < total_size * 0.9:
-            # Check the content of the buffer if its HTML or binary
-            if response.headers.get('content-type', '').startswith('text/html'):
-                logger.warn(f"Failed to download content for {link}. Found HTML content instead.")
-                return None
-        return buffer
-    except requests.exceptions.RequestException as e:
-        logger.error_trace(f"Failed to download from {link}: {e}")
-        return None
-
-def get_absolute_url(base_url: str, url: str) -> str:
-    """Get absolute URL from relative URL and base URL.
-    
-    Args:
-        base_url: Base URL
-        url: Relative URL
-    """
-    if url.strip() == "":
-        return ""
-    if url.strip("#") == "":
-        return ""
-    if url.startswith("http"):
-        return url
-    parsed_url = urlparse(url)
-    parsed_base = urlparse(base_url)
-    if parsed_url.netloc == "" or parsed_url.scheme == "":
-        parsed_url = parsed_url._replace(netloc=parsed_base.netloc, scheme=parsed_base.scheme)
-    return parsed_url.geturl()

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/cwa-book-downloader
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/cwa-bd_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+
+    # Cleanup any existing files or folders in the log directory
+    rm -rf "$LOG_DIR"/*
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,10 +56,16 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
@@ -28,34 +78,57 @@ if [ "$TZ" ]; then
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 fi
 
-# Set UID if not set
-if [ -z "$UID" ]; then
-    UID=1000
+# Determine user ID with proper precedence:
+# 1. PUID (LinuxServer.io standard - recommended)
+# 2. UID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+#
+# Note: $UID is a bash builtin that's always set. We use `printenv` to detect
+# if UID was explicitly set as an environment variable (e.g., via docker-compose).
+if [ -n "$PUID" ]; then
+    RUN_UID="$PUID"
+    echo "Using PUID=$RUN_UID"
+elif printenv UID >/dev/null 2>&1; then
+    RUN_UID="$(printenv UID)"
+    echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)"
+else
+    RUN_UID=1000
+    echo "Using default UID=$RUN_UID"
 fi
 
-# Set GID if not set
-if [ -z "$GID" ]; then
-    GID=100
+# Determine group ID with proper precedence:
+# 1. PGID (LinuxServer.io standard - recommended)
+# 2. GID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+if [ -n "$PGID" ]; then
+    RUN_GID="$PGID"
+    echo "Using PGID=$RUN_GID"
+elif [ -n "$GID" ]; then
+    RUN_GID="$GID"
+    echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)"
+else
+    RUN_GID=1000
+    echo "Using default GID=$RUN_GID"
 fi
 
-if ! getent group "$GID" >/dev/null; then
-    echo "Adding group $GID with name appuser"
-    groupadd -g "$GID" appuser
+if ! getent group "$RUN_GID" >/dev/null; then
+    echo "Adding group $RUN_GID with name appuser"
+    groupadd -g "$RUN_GID" appuser
 fi
 
 # Create user if it doesn't exist
-if ! id -u "$UID" >/dev/null 2>&1; then
-    echo "Adding user $UID with name appuser"
-    useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser
+if ! id -u "$RUN_UID" >/dev/null 2>&1; then
+    echo "Adding user $RUN_UID with name appuser"
+    useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser
 fi
 
 # Get username for the UID (whether we just created it or it existed)
-USERNAME=$(getent passwd "$UID" | cut -d: -f1)
-echo "Username for UID $UID is $USERNAME"
+USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1)
+echo "Username for UID $RUN_UID is $USERNAME"
 
 test_write() {
     folder=$1
-    test_file=$folder/calibre-web-automated-book-downloader_TEST_WRITE
+    test_file=$folder/shelfmark_TEST_WRITE
     mkdir -p $folder
     (
         echo 0123456789_TEST | sudo -E -u "$USERNAME" HOME=/app tee $test_file > /dev/null
@@ -75,6 +148,7 @@ test_write() {
 
 make_writable() {
     folder=$1
+    did_full_chown=0
     set +e
     test_write $folder
     is_writable=$?
@@ -84,40 +158,116 @@ make_writable() {
     else
         echo "Folder $folder is not writable, changing ownership"
         change_ownership $folder
-        chmod g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        chmod -R g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        did_full_chown=1
+    fi
+    # Fix any misowned subdirectories/files (e.g., from previous runs as root)
+    if [ "$did_full_chown" -eq 0 ] && [ -d "$folder" ]; then
+        echo "Checking for misowned files/directories in $folder"
+        # Stay on the same filesystem to avoid traversing mounted subpaths
+        # (for example read-only bind mounts under /app in dev setups).
+        find "$folder" -xdev -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+            -exec chown "$RUN_UID:$RUN_GID" {} + 2>/dev/null || true
     fi
     test_write $folder || echo "Failed to test write to ${folder}, continuing..."
 }
 
+fix_misowned() {
+    folder=$1
+    mkdir -p $folder
+    echo "Checking for misowned files/directories in $folder"
+    # Stay on the same filesystem to avoid traversing mounted subpaths
+    # (for example read-only bind mounts under /app in dev setups).
+    find "$folder" -xdev \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+        -exec chown "$RUN_UID:$RUN_GID" {} + 2>/dev/null || true
+}
+
 # Ensure proper ownership of application directories
 change_ownership() {
   folder=$1
   mkdir -p $folder
-  echo "Changing ownership of $folder to $USERNAME:$GID"
-  chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
-  chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
+  echo "Changing ownership of $folder to $USERNAME:$RUN_GID"
+  chown -R "${RUN_UID}:${RUN_GID}" "${folder}" || echo "Failed to change ownership for ${folder}, continuing..."
 }
 
-change_ownership /app
-change_ownership /var/log/cwa-book-downloader
-change_ownership /tmp/cwa-book-downloader
+fix_misowned /app
+fix_misowned /var/log/shelfmark
+fix_misowned /tmp/shelfmark
+
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
+
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
+fi
 
 # Test write to all folders
-make_writable /cwa-book-ingest
+make_writable ${CONFIG_DIR:-/config}
+make_writable ${INGEST_DIR:-/books}
 
-# Set the command to run based on DEBUG setting
-# DEBUG=true uses Flask dev server, otherwise uses gunicorn for production
-is_debug=$(echo "$DEBUG" | tr '[:upper:]' '[:lower:]')
-if [ "$is_debug" = "true" ]; then
-    command="python3 app.py"
-else
-    # Use geventwebsocket worker for SocketIO + WebSocket compatibility
-    # This special worker class handles WebSocket upgrades properly
-    # --workers 1: SocketIO requires sticky sessions, use 1 worker or configure sticky sessions
-    # -t 300: 300 second timeout for long-running requests
-    command="gunicorn --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} app:app"
+# Fix permissions on directories configured in settings
+echo "Checking for additional configured directories..."
+if [ -f /app/scripts/fix_permissions.py ]; then
+    configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
+    if [ -n "$configured_dirs" ]; then
+        echo "$configured_dirs" | while read -r dir; do
+            if [ -n "$dir" ] && [ -d "$dir" ]; then
+                echo "Checking configured directory: $dir"
+                make_writable "$dir"
+            fi
+        done
+    fi
 fi
 
+# Fallback to root if config dir is still not writable (common on NAS/Unraid after upgrade from v0.4.0)
+CONFIG_PATH=${CONFIG_DIR:-/config}
+set +e
+test_write "$CONFIG_PATH" >/dev/null 2>&1
+config_ok=$?
+set -e
+
+if [ $config_ok -ne 0 ] && [ "$RUN_UID" != "0" ]; then
+    config_owner=$(stat -c '%u' "$CONFIG_PATH" 2>/dev/null || echo "unknown")
+    if [ "$config_owner" = "0" ]; then
+        echo ""
+        echo "========================================================"
+        echo "WARNING: Permission issue detected!"
+        echo ""
+        echo "Config directory is owned by root but PUID=$RUN_UID."
+        echo "This typically happens after upgrading from v0.4.0 where"
+        echo "PUID/PGID settings were not respected."
+        echo ""
+        echo "Falling back to running as root to prevent data loss."
+        echo ""
+        echo "To fix this permanently, run on your HOST machine:"
+        echo "  chown -R $RUN_UID:$RUN_GID /path/to/config"
+        echo ""
+        echo "Then restart the container."
+        echo "========================================================"
+        echo ""
+        RUN_UID=0
+        RUN_GID=0
+        USERNAME=root
+    fi
+fi
+
+# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
+# upgrades work reliably on customer machines.
+# Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
+gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
+command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
+
 # If DEBUG and not using an external bypass
 if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     set +e
@@ -172,18 +322,24 @@ if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
 fi
 
-# Hacky way to verify /tmp has at least 1MB of space and is writable/readable
+# Verify /tmp has at least 1MB of space and is writable/readable
 echo "Verifying /tmp has enough space"
-rm -f /tmp/test.cwa-bd
-for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd
-sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))")
-[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1)
-rm /tmp/test.cwa-bd
+rm -f /tmp/test.shelfmark
+if dd if=/dev/zero of=/tmp/test.shelfmark bs=1M count=1 2>/dev/null && \
+   [ "$(wc -c < /tmp/test.shelfmark)" -eq 1048576 ]; then
+    rm -f /tmp/test.shelfmark
+    echo "Success: /tmp is writable and readable"
+else
+    echo "Failure: /tmp is not writable or has insufficient space"
+    exit 1
+fi
 
 echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
+# Set umask for file permissions (default: 0022 = files 644, dirs 755)
+UMASK_VALUE=${UMASK:-0022}
+echo "Setting umask to $UMASK_VALUE"
+umask $UMASK_VALUE
 
+stop_file_logging
 exec sudo -E -u "$USERNAME" HOME=/app $command

env.py

@@ -1,93 +0,0 @@
-import os
-from pathlib import Path
-
-def string_to_bool(s: str) -> bool:
-    return s.lower() in ["true", "yes", "1", "y"]
-
-# Authentication and session settings
-SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
-
-CWA_DB = os.getenv("CWA_DB_PATH")
-CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
-LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
-LOG_DIR = LOG_ROOT / "cwa-book-downloader"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
-INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
-INGEST_DIR_BOOK_FICTION = os.getenv("INGEST_DIR_BOOK_FICTION", "")
-INGEST_DIR_BOOK_NON_FICTION = os.getenv("INGEST_DIR_BOOK_NON_FICTION", "")
-INGEST_DIR_BOOK_UNKNOWN = os.getenv("INGEST_DIR_BOOK_UNKNOWN", "")
-INGEST_DIR_MAGAZINE = os.getenv("INGEST_DIR_MAGAZINE", "")
-INGEST_DIR_COMIC_BOOK = os.getenv("INGEST_DIR_COMIC_BOOK", "")
-INGEST_DIR_AUDIOBOOK = os.getenv("INGEST_DIR_AUDIOBOOK", "")
-INGEST_DIR_STANDARDS_DOCUMENT = os.getenv("INGEST_DIR_STANDARDS_DOCUMENT", "")
-INGEST_DIR_MUSICAL_SCORE = os.getenv("INGEST_DIR_MUSICAL_SCORE", "")
-INGEST_DIR_OTHER = os.getenv("INGEST_DIR_OTHER", "")
-DOWNLOAD_PATHS = {
-    "book (fiction)": Path(INGEST_DIR_BOOK_FICTION) if INGEST_DIR_BOOK_FICTION else INGEST_DIR,
-    "book (non-fiction)": Path(INGEST_DIR_BOOK_NON_FICTION) if INGEST_DIR_BOOK_NON_FICTION else INGEST_DIR,
-    "book (unknown)": Path(INGEST_DIR_BOOK_UNKNOWN) if INGEST_DIR_BOOK_UNKNOWN else INGEST_DIR,
-    "magazine": Path(INGEST_DIR_MAGAZINE) if INGEST_DIR_MAGAZINE else INGEST_DIR,
-    "comic book": Path(INGEST_DIR_COMIC_BOOK) if INGEST_DIR_COMIC_BOOK else INGEST_DIR,
-    "audiobook": Path(INGEST_DIR_AUDIOBOOK) if INGEST_DIR_AUDIOBOOK else INGEST_DIR,
-    "standards document": Path(INGEST_DIR_STANDARDS_DOCUMENT) if INGEST_DIR_STANDARDS_DOCUMENT else INGEST_DIR,
-    "musical score": Path(INGEST_DIR_MUSICAL_SCORE) if INGEST_DIR_MUSICAL_SCORE else INGEST_DIR,
-    "other": Path(INGEST_DIR_OTHER) if INGEST_DIR_OTHER else INGEST_DIR,
-}
-
-STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
-USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
-MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
-DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
-USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
-HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
-HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
-AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
-_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
-_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
-_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
-_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
-_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
-FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
-FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
-PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
-ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
-
-# Version information from Docker build
-BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
-RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
-
-# If debug is true, we want to log everything
-if DEBUG:
-    LOG_LEVEL = "DEBUG"
-else:
-    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
-MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
-MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "5"))
-DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip()
-USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
-BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
-
-# Logging settings
-LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
-
-USING_EXTERNAL_BYPASSER = string_to_bool(os.getenv("USING_EXTERNAL_BYPASSER", "false"))
-if USING_EXTERNAL_BYPASSER:
-    EXT_BYPASSER_URL = os.getenv("EXT_BYPASSER_URL", "http://flaresolverr:8191").strip()
-    EXT_BYPASSER_PATH = os.getenv("EXT_BYPASSER_PATH", "/v1").strip()
-    EXT_BYPASSER_TIMEOUT = int(os.getenv("EXT_BYPASSER_TIMEOUT", "60000"))
-
-USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
-# If using Tor, we don't need to set custom DNS, use DOH, or proxy
-if USING_TOR:
-    _CUSTOM_DNS = ""
-    USE_DOH = False
-    HTTP_PROXY = ""
-    HTTPS_PROXY = ""
-
-# Calibre-Web URL for navigation button
-CALIBRE_WEB_URL = os.getenv("CALIBRE_WEB_URL", "").strip()
-    

genDebug.sh

@@ -2,8 +2,8 @@
 
 # Set up log paths
 LOG_ROOT=${LOG_ROOT:-"/var/log"}
-LOG_DIR="$LOG_ROOT/cwa-book-downloader"
-OUTPUT_FILE_NAME="cwa-book-downloader-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
+LOG_DIR="$LOG_ROOT/shelfmark"
+OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
 OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
 
 # Create LOG_DIR if it doesn't exist
@@ -18,17 +18,17 @@ echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add disk usage
 echo "=== Disk Usage ===" >> "$LOG_DIR/system_info.txt"
-df -h >> "$LOG_DIR/system_info.txt"
+df -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add memory info
 echo "=== Memory Info ===" >> "$LOG_DIR/system_info.txt"
-free -h >> "$LOG_DIR/system_info.txt"
+free -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add running processes
 echo "=== Running Processes ===" >> "$LOG_DIR/system_info.txt"
-ps aux >> "$LOG_DIR/system_info.txt"
+ps aux >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add network information using basic commands
@@ -37,17 +37,17 @@ echo "=== Network Information ===" > "$LOG_DIR/network_info.txt"
 # Try to get basic connectivity information
 echo "=== Basic Connectivity ===" >> "$LOG_DIR/network_info.txt"
 echo "Hostname resolution:" >> "$LOG_DIR/network_info.txt"
-cat /etc/hosts 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
+cat /etc/hosts >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 echo "DNS configuration:" >> "$LOG_DIR/network_info.txt"
-cat /etc/resolv.conf 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
+cat /etc/resolv.conf >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try to get interface information from /proc
 echo "=== Network Interfaces (/proc) ===" >> "$LOG_DIR/network_info.txt"
 if [ -f "/proc/net/dev" ]; then
-  cat /proc/net/dev >> "$LOG_DIR/network_info.txt"
+  cat /proc/net/dev >> "$LOG_DIR/network_info.txt" 2>&1
 else
   echo "Not available: /proc/net/dev not found" >> "$LOG_DIR/network_info.txt"
 fi
@@ -55,9 +55,9 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try connectivity tests
 echo "=== Internet Connectivity ===" >> "$LOG_DIR/network_info.txt"
-ping -c 3 1.1.1.1 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
+ping -c 3 1.1.1.1 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
-ping -c 3 one.one.one.one 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
+ping -c 3 one.one.one.one >> "$LOG_DIR/network_info.txt" 2>&1 || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test IPv6 connectivity
@@ -77,7 +77,7 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try IPv6 connectivity test using Cloudflare's IPv6 DNS
 echo "Testing IPv6 connectivity to Cloudflare DNS:" >> "$LOG_DIR/network_info.txt"
-ping6 -c 3 2606:4700:4700::1111 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
+ping6 -c 3 2606:4700:4700::1111 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test SSL connectivity
@@ -92,24 +92,36 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Add installed packages
 echo "=== Installed Python Packages ===" > "$LOG_DIR/packages.txt"
-pip list 2>/dev/null >> "$LOG_DIR/packages.txt" || echo "pip not found" >> "$LOG_DIR/packages.txt"
+pip list >> "$LOG_DIR/packages.txt" 2>&1 || echo "pip not found" >> "$LOG_DIR/packages.txt"
 echo "" >> "$LOG_DIR/packages.txt"
 
 # Check Permissions
 echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
-ls -all /app >> "$LOG_DIR/permissions.txt"
+ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt"
+echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
+ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 
+# Check Iptables (NAT)
+echo "=== IPtables NAT Rules ===" > "$LOG_DIR/iptables_nat.txt"
+iptables -t nat -L -v -n >> "$LOG_DIR/iptables_nat.txt" 2>&1
+
+# Check DNS Resolution details
+echo "=== DNS Resolution Test ===" > "$LOG_DIR/dns_test.txt"
+echo "Resolving google.com:" >> "$LOG_DIR/dns_test.txt"
+nslookup google.com >> "$LOG_DIR/dns_test.txt" 2>&1
+echo "" >> "$LOG_DIR/dns_test.txt"
+echo "Resolving check.torproject.org:" >> "$LOG_DIR/dns_test.txt"
+nslookup check.torproject.org >> "$LOG_DIR/dns_test.txt" 2>&1
+
 
 # Check if running in Docker
 echo "=== Container Info ===" > "$LOG_DIR/container_info.txt"
@@ -122,19 +134,58 @@ else
 fi
 
 # Add environment variables (redacting sensitive info)
-env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt"
+env | grep -v -E "(AA_DONATOR_KEY|HARDCOVER_API_KEY|_KEY=|_SECRET=|_PASSWORD=|_TOKEN=)" | sort > "$LOG_DIR/environment.txt"
 
-echo "--- HTTPBin ---" > $LOG_DIR/network_info.txt
-curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt
-ehco ""
+# Add configuration files (redacting sensitive values)
+CONFIG_DIR=${CONFIG_DIR:-"/config"}
+if [ -d "$CONFIG_DIR" ]; then
+  mkdir -p "$LOG_DIR/config"
+
+  # Copy and redact main settings file
+  if [ -f "$CONFIG_DIR/settings.json" ]; then
+    # Redact sensitive fields (API keys, passwords, tokens)
+    sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+      "$CONFIG_DIR/settings.json" > "$LOG_DIR/config/settings.json" 2>/dev/null
+  fi
+
+  # Copy and redact plugin config files
+  if [ -d "$CONFIG_DIR/plugins" ]; then
+    mkdir -p "$LOG_DIR/config/plugins"
+    for config_file in "$CONFIG_DIR/plugins"/*.json; do
+      if [ -f "$config_file" ]; then
+        filename=$(basename "$config_file")
+        sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+          "$config_file" > "$LOG_DIR/config/plugins/$filename" 2>/dev/null
+      fi
+    done
+  fi
+
+  echo "Configuration files copied (sensitive values redacted)" >> "$LOG_DIR/container_info.txt"
+else
+  echo "Config directory not found at $CONFIG_DIR" >> "$LOG_DIR/container_info.txt"
+fi
+
+echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
+curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
-curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt
-ehco ""
+curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
-curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt
-ehco ""
+curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
-curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt
+curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt 2>&1
+
+# Copy Tor logs if they exist
+if [ -f "/var/log/tor/notices.log" ]; then
+  cp "/var/log/tor/notices.log" "$LOG_DIR/tor_notices.log"
+fi
+
+# Copy Supervisor logs if they exist
+if [ -d "/var/log/supervisor" ]; then
+  cp -rf "/var/log/supervisor/" "$LOG_DIR/supervisor/"
+fi
 
 # Create the zip file directly from LOG_DIR
 ln -s "$LOG_DIR" /tmp/$OUTPUT_FILE_NAME

models.py

@@ -1,356 +0,0 @@
-"""Data structures and models used across the application."""
-
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Tuple
-from enum import Enum
-from datetime import datetime, timedelta
-from threading import Lock, Event
-from pathlib import Path
-import queue
-import time
-from env import INGEST_DIR, STATUS_TIMEOUT
-
-class QueueStatus(str, Enum):
-    """Enum for possible book queue statuses."""
-    QUEUED = "queued"
-    RESOLVING = "resolving"
-    BYPASSING = "bypassing"
-    DOWNLOADING = "downloading"
-    VERIFYING = "verifying"
-    INGESTING = "ingesting"
-    COMPLETE = "complete"
-    AVAILABLE = "available"
-    ERROR = "error"
-    DONE = "done"
-    CANCELLED = "cancelled"
-
-@dataclass
-class QueueItem:
-    """Queue item with priority and metadata."""
-    book_id: str
-    priority: int
-    added_time: float
-    
-    def __lt__(self, other):
-        """Compare items for priority queue (lower priority number = higher precedence)."""
-        if self.priority != other.priority:
-            return self.priority < other.priority
-        return self.added_time < other.added_time
-
-@dataclass
-class BookInfo:
-    """Data class representing book information."""
-    id: str
-    title: str
-    preview: Optional[str] = None
-    author: Optional[str] = None
-    publisher: Optional[str] = None
-    year: Optional[str] = None
-    language: Optional[str] = None
-    content: Optional[str] = None
-    format: Optional[str] = None
-    size: Optional[str] = None
-    info: Optional[Dict[str, List[str]]] = None
-    description: Optional[str] = None
-    download_urls: List[str] = field(default_factory=list)
-    download_path: Optional[str] = None
-    priority: int = 0
-    progress: Optional[float] = None
-
-class BookQueue:
-    """Thread-safe book queue manager with priority support and cancellation."""
-    def __init__(self) -> None:
-        self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
-        self._lock = Lock()
-        self._status: dict[str, QueueStatus] = {}
-        self._book_data: dict[str, BookInfo] = {}
-        self._status_timestamps: dict[str, datetime] = {}  # Track when each status was last updated
-        self._status_timeout = timedelta(seconds=STATUS_TIMEOUT)  # 1 hour timeout
-        self._cancel_flags: dict[str, Event] = {}  # Cancellation flags for active downloads
-        self._active_downloads: dict[str, bool] = {}  # Track currently downloading books
-    
-    def add(self, book_id: str, book_data: BookInfo, priority: int = 0) -> None:
-        """Add a book to the queue with specified priority.
-        
-        Args:
-            book_id: Unique identifier for the book
-            book_data: Book information
-            priority: Priority level (lower number = higher priority)
-        """
-        with self._lock:
-            # Don't add if already exists and not in error/done state
-            if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                return
-                
-            book_data.priority = priority
-            queue_item = QueueItem(book_id, priority, time.time())
-            self._queue.put(queue_item)
-            self._book_data[book_id] = book_data
-            self._update_status(book_id, QueueStatus.QUEUED)
-    
-    def get_next(self) -> Optional[Tuple[str, Event]]:
-        """Get next book ID from queue with cancellation flag.
-        
-        Returns:
-            Tuple of (book_id, cancel_flag) or None if queue is empty
-        """
-        try:
-            queue_item = self._queue.get_nowait()
-            book_id = queue_item.book_id
-            
-            with self._lock:
-                # Check if book was cancelled while in queue
-                if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED:
-                    return self.get_next()  # Recursively get next non-cancelled item
-                
-                # Create cancellation flag for this download
-                cancel_flag = Event()
-                self._cancel_flags[book_id] = cancel_flag
-                self._active_downloads[book_id] = True
-                
-            return book_id, cancel_flag
-        except queue.Empty:
-            return None
-            
-    def _update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Internal method to update status and timestamp."""
-        self._status[book_id] = status
-        self._status_timestamps[book_id] = datetime.now()
-            
-    def update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Update status of a book in the queue."""
-        with self._lock:
-            self._update_status(book_id, status)
-            
-            # Clean up active download tracking when finished
-            if status in [QueueStatus.COMPLETE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                self._active_downloads.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-    
-    def update_download_path(self, book_id: str, download_path: str) -> None:
-        """Update the download path of a book in the queue."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].download_path = download_path
-                
-    def update_progress(self, book_id: str, progress: float) -> None:
-        """Update download progress for a book."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].progress = progress
-            
-    def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]:
-        """Get current queue status."""
-        self.refresh()
-        with self._lock:
-            result: Dict[QueueStatus, Dict[str, BookInfo]] = {status: {} for status in QueueStatus}
-            for book_id, status in self._status.items():
-                if book_id in self._book_data:
-                    result[status][book_id] = self._book_data[book_id]
-            return result
-            
-    def get_queue_order(self) -> List[Dict[str, any]]:
-        """Get current queue order for display."""
-        with self._lock:
-            queue_items = []
-            
-            # Get items from priority queue without removing them
-            temp_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    temp_items.append(item)
-                    if item.book_id in self._book_data:
-                        book_info = self._book_data[item.book_id]
-                        queue_items.append({
-                            'id': item.book_id,
-                            'title': book_info.title,
-                            'author': book_info.author,
-                            'priority': item.priority,
-                            'added_time': item.added_time,
-                            'status': self._status.get(item.book_id, QueueStatus.QUEUED)
-                        })
-                except queue.Empty:
-                    break
-            
-            # Put items back in queue
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
-            
-    def cancel_download(self, book_id: str) -> bool:
-        """Cancel a download and mark it as cancelled.
-        
-        Args:
-            book_id: Book identifier to cancel
-            
-        Returns:
-            bool: True if cancellation was successful
-        """
-        with self._lock:
-            current_status = self._status.get(book_id)
-            
-            # Allow cancellation during any active state
-            if current_status in [QueueStatus.RESOLVING, QueueStatus.BYPASSING, QueueStatus.DOWNLOADING, QueueStatus.VERIFYING, QueueStatus.INGESTING]:
-                # Signal active download to stop
-                if book_id in self._cancel_flags:
-                    self._cancel_flags[book_id].set()
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            elif current_status == QueueStatus.QUEUED:
-                # Remove from queue and mark as cancelled
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            
-            return False
-            
-    def set_priority(self, book_id: str, new_priority: int) -> bool:
-        """Change the priority of a queued book.
-        
-        Args:
-            book_id: Book identifier
-            new_priority: New priority level (lower = higher priority)
-            
-        Returns:
-            bool: True if priority was successfully changed
-        """
-        with self._lock:
-            if book_id not in self._status or self._status[book_id] != QueueStatus.QUEUED:
-                return False
-                
-            # Remove book from queue and re-add with new priority
-            temp_items = []
-            found = False
-            
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    if item.book_id == book_id:
-                        # Create new item with updated priority
-                        new_item = QueueItem(book_id, new_priority, item.added_time)
-                        temp_items.append(new_item)
-                        found = True
-                        # Update book data priority
-                        if book_id in self._book_data:
-                            self._book_data[book_id].priority = new_priority
-                    else:
-                        temp_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return found
-            
-    def reorder_queue(self, book_priorities: Dict[str, int]) -> bool:
-        """Bulk reorder queue by setting new priorities.
-        
-        Args:
-            book_priorities: Dict mapping book_id to new priority
-            
-        Returns:
-            bool: True if reordering was successful
-        """
-        with self._lock:
-            # Extract all items from queue
-            all_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    # Update priority if specified
-                    if item.book_id in book_priorities:
-                        new_priority = book_priorities[item.book_id]
-                        item = QueueItem(item.book_id, new_priority, item.added_time)
-                        # Update book data priority
-                        if item.book_id in self._book_data:
-                            self._book_data[item.book_id].priority = new_priority
-                    all_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back with updated priorities
-            for item in all_items:
-                self._queue.put(item)
-                
-            return True
-            
-    def get_active_downloads(self) -> List[str]:
-        """Get list of currently active download book IDs."""
-        with self._lock:
-            return list(self._active_downloads.keys())
-            
-    def clear_completed(self) -> int:
-        """Remove all completed, errored, or cancelled books from tracking.
-        
-        Returns:
-            int: Number of books removed
-        """
-        with self._lock:
-            to_remove = []
-            for book_id, status in self._status.items():
-                if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
-                    to_remove.append(book_id)
-            
-            removed_count = len(to_remove)
-            for book_id in to_remove:
-                self._status.pop(book_id, None)
-                self._status_timestamps.pop(book_id, None)
-                self._book_data.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-                self._active_downloads.pop(book_id, None)
-                
-            return removed_count
-        
-    def refresh(self) -> None:
-        """Remove any books that are done downloading or have stale status."""
-        with self._lock:
-            current_time = datetime.now()
-            
-            # Create a list of items to remove to avoid modifying dict during iteration
-            to_remove = []
-            
-            for book_id, status in self._status.items():
-                path = self._book_data[book_id].download_path
-                if path and not Path(path).exists():
-                    self._book_data[book_id].download_path = None
-                    path = None
-                
-                # Check for completed downloads
-                if status == QueueStatus.AVAILABLE:
-                    if not path:
-                        self._update_status(book_id, QueueStatus.DONE)
-                
-                # Check for stale status entries
-                last_update = self._status_timestamps.get(book_id)
-                if last_update and (current_time - last_update) > self._status_timeout:
-                    if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]:
-                        to_remove.append(book_id)
-            
-            # Remove stale entries
-            for book_id in to_remove:
-                del self._status[book_id]
-                del self._status_timestamps[book_id]
-                if book_id in self._book_data:
-                    del self._book_data[book_id]
-
-    def set_status_timeout(self, hours: int) -> None:
-        """Set the status timeout duration in hours."""
-        with self._lock:
-            self._status_timeout = timedelta(hours=hours)
-
-
-# Global instance of BookQueue
-book_queue = BookQueue()
-
-@dataclass
-class SearchFilters:
-    isbn: Optional[List[str]] = None
-    author: Optional[List[str]] = None
-    title: Optional[List[str]] = None
-    lang: Optional[List[str]] = None
-    sort: Optional[str] = None
-    content: Optional[List[str]] = None
-    format: Optional[List[str]] = None

network.py

@@ -1,350 +0,0 @@
-"""Network operations manager for the book downloader application."""
-
-import requests
-import urllib.request
-from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable
-import socket
-import dns.resolver
-from socket import AddressFamily, SocketKind
-import urllib.parse
-import ssl
-import ipaddress
-
-from logger import setup_logger
-from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER
-import config
-
-logger = setup_logger(__name__)
-
-# Common helper functions for DNS resolution
-def _decode_host(host: Union[str, bytes, None]) -> str:
-    """Convert host to string, handling bytes and None cases."""
-    if host is None:
-        return ""
-    if isinstance(host, bytes):
-        return host.decode('utf-8')
-    return str(host)
-
-def _decode_port(port: Union[str, bytes, int, None]) -> int:
-    """Convert port to integer, handling various input types."""
-    if port is None:
-        return 0
-    if isinstance(port, (str, bytes)):
-        return int(port)
-    return int(port)
-
-def _is_local_address(host_str: str) -> bool:
-    """Check if an address is local and should bypass custom DNS."""
-    """Check if an address is local or private and should bypass custom DNS."""
-    # Localhost checks
-    if (host_str == 'localhost' or 
-        host_str.startswith('127.') or 
-        host_str == '::1' or 
-        host_str == '0.0.0.0'):
-        return True
-        
-    # IPv4 private ranges (RFC 1918)
-    if (host_str.startswith('10.') or 
-        (host_str.startswith('172.') and 
-         len(host_str.split('.')) > 1 and 
-         16 <= int(host_str.split('.')[1]) <= 31) or
-        host_str.startswith('192.168.')):
-        return True
-        
-    # IPv6 private ranges
-    if (host_str.startswith('fc') or 
-        host_str.startswith('fd') or  # Unique local addresses (fc00::/7)
-        host_str.startswith('fe80:')):  # Link-local addresses (fe80::/10)
-        return True
-    
-    return False
-
-def _is_ip_address(host_str: str) -> bool:
-    """Check if a string is a valid IP address (IPv4 or IPv6)."""
-    try:
-        ipaddress.ip_address(host_str)
-        return True
-    except ValueError:
-        return False
-
-# Store the original getaddrinfo function
-original_getaddrinfo = socket.getaddrinfo
-
-class DoHResolver:
-    """DNS over HTTPS resolver implementation."""
-    def __init__(self, provider_url: str, hostname: str, ip: str):
-        """Initialize DoH resolver with specified provider."""
-        self.base_url = provider_url.lower().strip()
-        self.hostname = hostname  # Store the hostname for hostname-based skipping
-        self.ip = ip              # Store IP for direct connections
-        self.session = requests.Session()
-        
-        # Different headers based on provider
-        if 'google' in self.base_url:
-            self.session.headers.update({
-                'Accept': 'application/json',
-            })
-        else:
-            self.session.headers.update({
-                'Accept': 'application/dns-json',
-            })
-    
-    def resolve(self, hostname: str, record_type: str) -> List[str]:
-        """Resolve a hostname using DoH.
-        
-        Args:
-            hostname: The hostname to resolve
-            record_type: The DNS record type (A or AAAA)
-            
-        Returns:
-            List of resolved IP addresses
-        """
-        # Check if hostname is already an IP address, no need to resolve
-        if _is_ip_address(hostname):
-            logger.debug(f"Skipping DoH resolution for IP address: {hostname}")
-            return [hostname]
-            
-        # Check if hostname is a private IP address, and skip DoH if it is
-        if _is_local_address(hostname):
-            logger.debug(f"Skipping DoH resolution for private IP: {hostname}")
-            return [hostname]
-            
-        # Skip resolution for the DoH server itself to prevent recursion
-        if hostname == self.hostname:
-            logger.debug(f"Skipping DoH resolution for DoH server itself: {hostname}")
-            return [self.ip]
-            
-        try:
-            params = {
-                'name': hostname,
-                'type': 'AAAA' if record_type == 'AAAA' else 'A'
-            }
-            
-            response = self.session.get(
-                self.base_url,
-                params=params,
-                proxies=PROXIES,
-                timeout=5
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            if 'Answer' not in data:
-                logger.warning(f"DoH resolution failed for {hostname}: {data}")
-                return []
-            
-            # Extract IP addresses from the response    
-            answers = [answer['data'] for answer in data['Answer'] 
-                    if answer.get('type') == (28 if record_type == 'AAAA' else 1)]
-            logger.debug(f"Resolved {hostname} to {len(answers)} addresses using DoH: {answers}")
-            return answers
-            
-        except Exception as e:
-            logger.warning(f"DoH resolution failed for {hostname}: {e}")
-            return []
-
-def create_custom_resolver():
-    """Create a custom DNS resolver using the configured DNS servers."""
-    custom_resolver = dns.resolver.Resolver()
-    custom_resolver.nameservers = CUSTOM_DNS
-    return custom_resolver
-
-def resolve_with_custom_dns(resolver, hostname: str, record_type: str) -> List[str]:
-    """Resolve hostname using custom DNS resolver.
-    
-    Args:
-        resolver: The DNS resolver to use
-        hostname: The hostname to resolve
-        record_type: The DNS record type (A or AAAA)
-        
-    Returns:
-        List of resolved IP addresses
-    """
-    try:
-        answers = resolver.resolve(hostname, record_type)
-        return [str(answer) for answer in answers]
-    except Exception as e:
-        logger.debug(f"{record_type} resolution failed for {hostname}: {e}")
-        return []
-
-def create_custom_getaddrinfo(
-    resolve_ipv4: Callable[[str], List[str]],
-    resolve_ipv6: Callable[[str], List[str]],
-    skip_check: Optional[Callable[[str], bool]] = None
-):
-    """Create a custom getaddrinfo function that uses the provided resolvers.
-    
-    Args:
-        resolve_ipv4: Function to resolve IPv4 addresses
-        resolve_ipv6: Function to resolve IPv6 addresses
-        skip_check: Optional function to check if custom resolution should be skipped
-        
-    Returns:
-        A custom getaddrinfo function
-    """
-    def custom_getaddrinfo(
-        host: Union[str, bytes, None],
-        port: Union[str, bytes, int, None],
-        family: int = 0,
-        type: int = 0,
-        proto: int = 0,
-        flags: int = 0
-    ) -> Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]]:
-        host_str = _decode_host(host)
-        port_int = _decode_port(port)
-        
-        # Skip custom resolution for IP addresses, local addresses, or if skip check passes
-        if _is_ip_address(host_str) or _is_local_address(host_str) or (skip_check and skip_check(host_str)):
-            logger.debug(f"Using system DNS for IP address or local/private address: {host_str}")
-            return original_getaddrinfo(host, port, family, type, proto, flags)
-        
-        results: list[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]] = []
-        
-        try:
-            # Try IPv6 first if family allows it
-            if family == 0 or family == socket.AF_INET6:
-                logger.debug(f"Resolving IPv6 address for {host_str}")
-                ipv6_answers = resolve_ipv6(host_str)
-                for answer in ipv6_answers:
-                    results.append((socket.AF_INET6, cast(SocketKind, type), proto, '', (answer, port_int, 0, 0)))
-                if ipv6_answers:
-                    logger.debug(f"Found {len(ipv6_answers)} IPv6 addresses for {host_str}")
-            
-            # Then try IPv4
-            if family == 0 or family == socket.AF_INET:
-                logger.debug(f"Resolving IPv4 address for {host_str}")
-                ipv4_answers = resolve_ipv4(host_str)
-                for answer in ipv4_answers:
-                    results.append((socket.AF_INET, cast(SocketKind, type), proto, '', (answer, port_int)))
-                if ipv4_answers:
-                    logger.debug(f"Found {len(ipv4_answers)} IPv4 addresses for {host_str}")
-            
-            if results:
-                logger.debug(f"Resolved {host_str} to {len(results)} addresses")
-                return results
-                
-        except Exception as e:
-            logger.warning(f"Custom DNS resolution failed for {host_str}: {e}, falling back to system DNS")
-        
-        # Fall back to system DNS if custom resolution fails
-        try:
-            return original_getaddrinfo(host, port, family, type, proto, flags)
-        except Exception as e:
-            logger.error(f"System DNS resolution also failed for {host_str}: {e}")
-            # Last resort: Try to connect to the hostname directly
-            if family == 0 or family == socket.AF_INET:
-                logger.warning(f"Using direct hostname as last resort for {host_str}")
-                return [(socket.AF_INET, cast(SocketKind, type), proto, '', (host_str, port_int))]
-            else:
-                raise  # Re-raise the exception if we can't provide a last resort
-    
-    return custom_getaddrinfo
-
-def init_doh_resolver(doh_server: str = DOH_SERVER):
-    """Initialize DNS over HTTPS resolver.
-    
-    Args:
-        doh_server: The DoH server URL
-    """
-    # Pre-resolve the DoH server hostname to prevent recursion
-    url = urllib.parse.urlparse(doh_server)
-    server_hostname = url.hostname if url.hostname else ''
-    
-    # Use system DNS for DoH server to prevent circular dependencies
-    try:
-        # Temporarily restore original getaddrinfo to resolve DoH server
-        temp_getaddrinfo = socket.getaddrinfo
-        socket.getaddrinfo = original_getaddrinfo
-        
-        server_ip = socket.gethostbyname(server_hostname)
-        logger.info(f"DoH server {server_hostname} resolved to IP: {server_ip}")
-        
-        # Restore custom getaddrinfo if it was previously set
-        socket.getaddrinfo = temp_getaddrinfo
-    except Exception as e:
-        logger.error(f"Failed to resolve DoH server {server_hostname}: {e}")
-        # Fall back to a known public DNS if resolution fails
-        server_ip = "1.1.1.1"
-        logger.info(f"Using fallback IP for DoH server: {server_ip}")
-    
-    # Create DoH resolver
-    doh_resolver = DoHResolver(doh_server, server_hostname, server_ip)
-    
-    # Create resolver functions
-    def resolve_ipv4(hostname: str) -> List[str]:
-        return doh_resolver.resolve(hostname, 'A')
-    
-    def resolve_ipv6(hostname: str) -> List[str]:
-        return doh_resolver.resolve(hostname, 'AAAA')
-    
-    # Skip DoH resolution for the DoH server itself, IP addresses, and private addresses
-    def skip_doh(hostname: str) -> bool:
-        return (hostname == server_hostname or 
-                hostname == server_ip or 
-                _is_ip_address(hostname) or 
-                _is_local_address(hostname))
-    
-    # Replace socket.getaddrinfo with our DoH-enabled version
-    socket.getaddrinfo = cast(Any, create_custom_getaddrinfo(
-        resolve_ipv4, resolve_ipv6, skip_doh
-    ))
-    
-    logger.info("DoH resolver successfully configured and activated")
-    return doh_resolver
-
-def init_custom_resolver():
-    """Initialize custom DNS resolver using configured DNS servers."""
-    custom_resolver = create_custom_resolver()
-    
-    # Create resolver functions
-    def resolve_ipv4(hostname: str) -> List[str]:
-        return resolve_with_custom_dns(custom_resolver, hostname, 'A')
-    
-    def resolve_ipv6(hostname: str) -> List[str]:
-        return resolve_with_custom_dns(custom_resolver, hostname, 'AAAA')
-    
-    # Replace socket.getaddrinfo with our custom resolver
-    socket.getaddrinfo = cast(Any, create_custom_getaddrinfo(resolve_ipv4, resolve_ipv6))
-    
-    logger.info("Custom DNS resolver successfully configured and activated")
-    return custom_resolver
-
-# Initialize DNS resolvers based on configuration
-def init_dns_resolvers():
-    """Initialize DNS resolvers based on configuration."""
-    if len(CUSTOM_DNS) > 0:
-        init_custom_resolver()
-        if DOH_SERVER:
-            init_doh_resolver()
-
-# Initialize DNS resolvers
-init_dns_resolvers()
-
-# Check available AA_BASE_URLs if set to auto
-if AA_BASE_URL == "auto":
-    logger.info(f"AA_BASE_URL: auto, checking available urls {AA_AVAILABLE_URLS}")
-    for url in AA_AVAILABLE_URLS:
-        try:
-            response = requests.get(url, proxies=PROXIES)
-            if response.status_code == 200:
-                AA_BASE_URL = url
-                break
-        except Exception as e:
-            logger.error_trace(f"Error checking {url}: {e}")
-    if AA_BASE_URL == "auto":
-        AA_BASE_URL = AA_AVAILABLE_URLS[0]
-config.AA_BASE_URL = AA_BASE_URL
-logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
-
-# Configure urllib opener with appropriate headers
-opener = urllib.request.build_opener()
-opener.addheaders = [
-    ('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        'AppleWebKit/537.36 (KHTML, like Gecko) '
-        'Chrome/129.0.0.0 Safari/537.3')
-]
-urllib.request.install_opener(opener)
-
-# Need an empty function to be called by downloader.py
-def init():
-    pass

node_modules/.bin/baseline-browser-mapping

@@ -0,0 +1 @@
+../baseline-browser-mapping/dist/cli.js

node_modules/.package-lock.json

@@ -0,0 +1,17 @@
+{
+  "name": "shelfmark",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "node_modules/baseline-browser-mapping": {
+      "version": "2.9.19",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz",
+      "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "baseline-browser-mapping": "dist/cli.js"
+      }
+    }
+  }
+}

node_modules/baseline-browser-mapping/LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

node_modules/baseline-browser-mapping/README.md

@@ -0,0 +1,463 @@
+# [`baseline-browser-mapping`](https://github.com/web-platform-dx/web-features/packages/baseline-browser-mapping)
+
+By the [W3C WebDX Community Group](https://www.w3.org/community/webdx/) and contributors.
+
+`baseline-browser-mapping` provides:
+
+- An `Array` of browsers compatible with Baseline Widely available and Baseline year feature sets via the [`getCompatibleVersions()` function](#get-baseline-widely-available-browser-versions-or-baseline-year-browser-versions).
+- An `Array`, `Object` or `CSV` as a string describing the Baseline feature set support of all browser versions included in the module's data set via the [`getAllVersions()` function](#get-data-for-all-browser-versions).
+
+You can use `baseline-browser-mapping` to help you determine minimum browser version support for your chosen Baseline feature set; or to analyse the level of support for different Baseline feature sets in your site's traffic by joining the data with your analytics data.
+
+## Install for local development
+
+To install the package, run:
+
+`npm install --save-dev baseline-browser-mapping`
+
+`baseline-browser-mapping` depends on `web-features` and `@mdn/browser-compat-data` for core browser version selection, but the data is pre-packaged and minified. This package checks for updates to those modules and the supported [downstream browsers](#downstream-browsers) on a daily basis and is updated frequently. Consider adding a script to your `package.json` to update `baseline-browser-mapping` and using it as part of your build process to ensure your data is as up to date as possible:
+
+```javascript
+"scripts": [
+  "refresh-baseline-browser-mapping": "npm i --save-dev baseline-browser-mapping@latest"
+]
+```
+
+The minimum supported NodeJS version for `baseline-browser-mapping` is v8 in alignment with `browserslist`. For NodeJS versions earlier than v13.2, the [`require('baseline-browser-mapping')`](https://nodejs.org/api/modules.html#requireid) syntax should be used to import the module.
+
+## Keeping `baseline-browser-mapping` up to date
+
+If you are only using this module to generate minimum browser versions for Baseline Widely available or Baseline year feature sets, you don't need to update this module frequently, as the backward looking data is reasonably stable.
+
+However, if you are targeting Newly available, using the [`getAllVersions()`](#get-data-for-all-browser-versions) function or heavily relying on the data for downstream browsers, you should update this module more frequently. If you target a feature cut off date within the last two months and your installed version of `baseline-browser-mapping` has data that is more than 2 months old, you will receive a console warning advising you to update to the latest version when you call `getCompatibleVersions()` or `getAllVersions()`.
+
+If you want to suppress these warnings you can use the `suppressWarnings: true` option in the configuration object passed to `getCompatibleVersions()` or `getAllVersions()`. Alternatively, you can use the `BASELINE_BROWSER_MAPPING_IGNORE_OLD_DATA=true` environment variable when running your build process. This module also respects the `BROWSERSLIST_IGNORE_OLD_DATA=true` environment variable. Environment variables can also be provided in a `.env` file from Node 20 onwards; however, this module does not load .env files automatically to avoid conflicts with other libraries with different requirements. You will need to use `process.loadEnvFile()` or a library like `dotenv` to load .env files before `baseline-browser-mapping` is called.
+
+If you want to ensure [reproducible builds](https://www.wikiwand.com/en/articles/Reproducible_builds), we strongly recommend using the `widelyAvailableOnDate` option to fix the Widely available date on a per build basis to ensure dependent tools provide the same output and you do not produce data staleness warnings. If you are using [`browserslist`](https://github.com/browserslist/browserslist) to target Baseline Widely available, consider automatically updating your `browserslist` configuration in `package.json` or `.browserslistrc` to `baseline widely available on {YYYY-MM-DD}` as part of your build process to ensure the same or sufficiently similar list of minimum browsers is reproduced for historical builds.
+
+## Importing `baseline-browser-mapping`
+
+This module exposes two functions: `getCompatibleVersions()` and `getAllVersions()`, both which can be imported directly from `baseline-browser-mapping`:
+
+```javascript
+import {
+  getCompatibleVersions,
+  getAllVersions,
+} from "baseline-browser-mapping";
+```
+
+If you want to load the script and data directly in a web page without hosting it yourself, consider using a CDN:
+
+```html
+<script type="module">
+  import {
+    getCompatibleVersions,
+    getAllVersions,
+  } from "https://cdn.jsdelivr.net/npm/baseline-browser-mapping";
+</script>
+```
+
+## Get Baseline Widely available browser versions or Baseline year browser versions
+
+To get the current list of minimum browser versions compatible with Baseline Widely available features from the core browser set, call the `getCompatibleVersions()` function:
+
+```javascript
+getCompatibleVersions();
+```
+
+Executed on 7th March 2025, the above code returns the following browser versions:
+
+```javascript
+[
+  { browser: "chrome", version: "105", release_date: "2022-09-02" },
+  {
+    browser: "chrome_android",
+    version: "105",
+    release_date: "2022-09-02",
+  },
+  { browser: "edge", version: "105", release_date: "2022-09-02" },
+  { browser: "firefox", version: "104", release_date: "2022-08-23" },
+  {
+    browser: "firefox_android",
+    version: "104",
+    release_date: "2022-08-23",
+  },
+  { browser: "safari", version: "15.6", release_date: "2022-09-02" },
+  {
+    browser: "safari_ios",
+    version: "15.6",
+    release_date: "2022-09-02",
+  },
+];
+```
+
+> [!NOTE]
+> The minimum versions of each browser are not necessarily the final release before the Widely available cutoff date of `TODAY - 30 MONTHS`. Some earlier versions will have supported the full Widely available feature set.
+
+### `getCompatibleVersions()` configuration options
+
+`getCompatibleVersions()` accepts an `Object` as an argument with configuration options. The defaults are as follows:
+
+```javascript
+{
+  targetYear: undefined,
+  widelyAvailableOnDate: undefined,
+  includeDownstreamBrowsers: false,
+  listAllCompatibleVersions: false,
+  suppressWarnings: false
+}
+```
+
+#### `targetYear`
+
+The `targetYear` option returns the minimum browser versions compatible with all **Baseline Newly available** features at the end of the specified calendar year. For example, calling:
+
+```javascript
+getCompatibleVersions({
+  targetYear: 2020,
+});
+```
+
+Returns the following versions:
+
+```javascript
+[
+  { browser: "chrome", version: "87", release_date: "2020-11-19" },
+  {
+    browser: "chrome_android",
+    version: "87",
+    release_date: "2020-11-19",
+  },
+  { browser: "edge", version: "87", release_date: "2020-11-19" },
+  { browser: "firefox", version: "83", release_date: "2020-11-17" },
+  {
+    browser: "firefox_android",
+    version: "83",
+    release_date: "2020-11-17",
+  },
+  { browser: "safari", version: "14", release_date: "2020-09-16" },
+  { browser: "safari_ios", version: "14", release_date: "2020-09-16" },
+];
+```
+
+> [!NOTE]
+> The minimum version of each browser is not necessarily the final version released in that calendar year. In the above example, Firefox 84 was the final version released in 2020; however Firefox 83 supported all of the features that were interoperable at the end of 2020.
+> [!WARNING]
+> You cannot use `targetYear` and `widelyAavailableDate` together. Please only use one of these options at a time.
+
+#### `widelyAvailableOnDate`
+
+The `widelyAvailableOnDate` option returns the minimum versions compatible with Baseline Widely available on a specified date in the format `YYYY-MM-DD`:
+
+```javascript
+getCompatibleVersions({
+  widelyAvailableOnDate: `2023-04-05`,
+});
+```
+
+> [!TIP]
+> This option is useful if you provide a versioned library that targets Baseline Widely available on each version's release date and you need to provide a statement on minimum supported browser versions in your documentation.
+
+#### `includeDownstreamBrowsers`
+
+Setting `includeDownstreamBrowsers` to `true` will include browsers outside of the Baseline core browser set where it is possible to map those browsers to an upstream Chromium or Gecko version:
+
+```javascript
+getCompatibleVersions({
+  includeDownstreamBrowsers: true,
+});
+```
+
+For more information on downstream browsers, see [the section on downstream browsers](#downstream-browsers) below.
+
+#### `includeKaiOS`
+
+KaiOS is an operating system and app framework based on the Gecko engine from Firefox. KaiOS is based on the Gecko engine and feature support can be derived from the upstream Gecko version that each KaiOS version implements. However KaiOS requires other considerations beyond feature compatibility to ensure a good user experience as it runs on device types that do not have either mouse and keyboard or touch screen input in the way that all the other browsers supported by this module do.
+
+```javascript
+getCompatibleVersions({
+  includeDownstreamBrowsers: true,
+  includeKaiOS: true,
+});
+```
+
+> [!NOTE]
+> Including KaiOS requires you to include all downstream browsers using the `includeDownstreamBrowsers` option.
+
+#### `listAllCompatibleVersions`
+
+Setting `listAllCompatibleVersions` to true will include the minimum versions of each compatible browser, and all the subsequent versions:
+
+```javascript
+getCompatibleVersions({
+  listAllCompatibleVersions: true,
+});
+```
+
+#### `suppressWarnings`
+
+Setting `suppressWarnings` to `true` will suppress the console warning about old data:
+
+```javascript
+getCompatibleVersions({
+  suppressWarnings: true,
+});
+```
+
+## Get data for all browser versions
+
+You may want to obtain data on all the browser versions available in this module for use in an analytics solution or dashboard. To get details of each browser version's level of Baseline support, call the `getAllVersions()` function:
+
+```javascript
+import { getAllVersions } from "baseline-browser-mapping";
+
+getAllVersions();
+```
+
+By default, this function returns an `Array` of `Objects` and excludes downstream browsers:
+
+```javascript
+[
+  ...
+  {
+    browser: "firefox_android", // Browser name
+    version: "125", // Browser version
+    release_date: "2024-04-16", // Release date
+    year: 2023, // Baseline year feature set the version supports
+    wa_compatible: true // Whether the browser version supports Widely available
+  },
+  ...
+]
+```
+
+For browser versions in `@mdn/browser-compat-data` that were released before Baseline can be defined, i.e. Baseline 2015, the `year` property is always the string: `"pre_baseline"`.
+
+### Understanding which browsers support Newly available features
+
+You may want to understand which recent browser versions support all Newly available features. You can replace the `wa_compatible` property with a `supports` property using the `useSupport` option:
+
+```javascript
+getAllVersions({
+  useSupports: true,
+});
+```
+
+The `supports` property is optional and has two possible values:
+
+- `widely` for browser versions that support all Widely available features.
+- `newly` for browser versions that support all Newly available features.
+
+Browser versions that do not support Widely or Newly available will not include the `support` property in the `array` or `object` outputs, and in the CSV output, the `support` column will contain an empty string. Browser versions that support all Newly available features also support all Widely available features.
+
+### `getAllVersions()` Configuration options
+
+`getAllVersions()` accepts an `Object` as an argument with configuration options. The defaults are as follows:
+
+```javascript
+{
+  includeDownstreamBrowsers: false,
+  outputFormat: "array",
+  suppressWarnings: false
+}
+```
+
+#### `includeDownstreamBrowsers` (in `getAllVersions()` output)
+
+As with `getCompatibleVersions()`, you can set `includeDownstreamBrowsers` to `true` to include the Chromium and Gecko downstream browsers [listed below](#list-of-downstream-browsers).
+
+```javascript
+getAllVersions({
+  includeDownstreamBrowsers: true,
+});
+```
+
+Downstream browsers include the same properties as core browsers, as well as the `engine`they use and `engine_version`, for example:
+
+```javascript
+[
+  ...
+  {
+    browser: "samsunginternet_android",
+    version: "27.0",
+    release_date: "2024-11-06",
+    engine: "Blink",
+    engine_version: "125",
+    year: 2023,
+    supports: "widely"
+  },
+  ...
+]
+```
+
+#### `includeKaiOS` (in `getAllVersions()` output)
+
+As with `getCompatibleVersions()` you can include KaiOS in your output. The same requirement to have `includeDownstreamBrowsers: true` applies.
+
+```javascript
+getAllVersions({
+  includeDownstreamBrowsers: true,
+  includeKaiOS: true,
+});
+```
+
+#### `suppressWarnings` (in `getAllVersions()` output)
+
+As with `getCompatibleVersions()`, you can set `suppressWarnings` to `true` to suppress the console warning about old data:
+
+```javascript
+getAllVersions({
+  suppressWarnings: true,
+});
+```
+
+#### `outputFormat`
+
+By default, this function returns an `Array` of `Objects` which can be manipulated in Javascript or output to JSON.
+
+To return an `Object` that nests keys , set `outputFormat` to `object`:
+
+```javascript
+getAllVersions({
+  outputFormat: "object",
+});
+```
+
+In thise case, `getAllVersions()` returns a nested object with the browser [IDs listed below](#list-of-downstream-browsers) as keys, and versions as keys within them:
+
+```javascript
+{
+  "chrome": {
+    "53": {
+      "year": 2016,
+      "release_date": "2016-09-07"
+    },
+    ...
+}
+```
+
+Downstream browsers will include extra fields for `engine` and `engine_versions`
+
+```javascript
+{
+  ...
+  "webview_android": {
+    "53": {
+      "year": 2016,
+      "release_date": "2016-09-07",
+      "engine": "Blink",
+      "engine_version": "53"
+    },
+  ...
+}
+```
+
+To return a `String` in CSV format, set `outputFormat` to `csv`:
+
+```javascript
+getAllVersions({
+  outputFormat: "csv",
+});
+```
+
+`getAllVersions` returns a `String` with a header row and comma-separated values for each browser version that you can write to a file or pass to another service. Core browsers will have "NULL" as the value for their `engine` and `engine_version`:
+
+```csv
+"browser","version","year","supports","release_date","engine","engine_version"
+...
+"chrome","24","pre_baseline","","2013-01-10","NULL","NULL"
+...
+"chrome","53","2016","","2016-09-07","NULL","NULL"
+...
+"firefox","135","2024","widely","2025-02-04","NULL","NULL"
+"firefox","136","2024","newly","2025-03-04","NULL","NULL"
+...
+"ya_android","20.12","2020","year_only","2020-12-20","Blink","87"
+...
+```
+
+> [!NOTE]
+> The above example uses `"includeDownstreamBrowsers": true`
+
+### Static resources
+
+The outputs of `getAllVersions()` are available as JSON or CSV files generated on a daily basis and hosted on GitHub pages:
+
+- Core browsers only
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_array.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_object.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions.csv)
+- Core browsers only, with `supports` property
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_array_with_supports.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_object_with_supports.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_with_supports.csv)
+- Including downstream browsers
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_array.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_object.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions.csv)
+- Including downstream browsers with `supports` property
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_array_with_supports.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_object_with_supports.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_with_supports.csv)
+
+These files are updated on a daily basis.
+
+## CLI
+
+`baseline-browser-mapping` includes a command line interface that exposes the same data and options as the `getCompatibleVersions()` function. To learn more about using the CLI, run:
+
+```sh
+npx baseline-browser-mapping --help
+```
+
+## Downstream browsers
+
+### Limitations
+
+The browser versions in this module come from two different sources:
+
+- MDN's `browser-compat-data` module.
+- Parsed user agent strings provided by [useragents.io](https://useragents.io/)
+
+MDN `browser-compat-data` is an authoritative source of information for the browsers it contains. The release dates for the Baseline core browser set and the mapping of downstream browsers to Chromium versions should be considered accurate.
+
+Browser mappings from useragents.io are provided on a best effort basis. They assume that browser vendors are accurately stating the Chromium version they have implemented. The initial set of version mappings was derived from a bulk export in November 2024. This version was iterated over with a Regex match looking for a major Chrome version and a corresponding version of the browser in question, e.g.:
+
+`Mozilla/5.0 (Linux; U; Android 10; en-US; STK-L21 Build/HUAWEISTK-L21) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/100.0.4896.58 UCBrowser/13.8.2.1324 Mobile Safari/537.36`
+
+Shows UC Browser Mobile 13.8 implementing Chromium 100, and:
+
+`Mozilla/5.0 (Linux; arm_64; Android 11; Redmi Note 8 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.123 YaBrowser/24.10.2.123.00 SA/3 Mobile Safari/537.36`
+
+Shows Yandex Browser Mobile 24.10 implementing Chromium 128. The Chromium version from this string is mapped to the corresponding Chrome version from MDN `browser-compat-data`.
+
+> [!NOTE]
+> Where possible, approximate release dates have been included based on useragents.io "first seen" data. useragents.io does not have "first seen" dates prior to June 2020. However, these browsers' Baseline compatibility is determined by their Chromium or Gecko version, so their release dates are more informative than critical.
+
+This data is updated on a daily basis using a [script](https://github.com/web-platform-dx/web-features/tree/main/scripts/refresh-downstream.ts) triggered by a GitHub [action](https://github.com/web-platform-dx/web-features/tree/main/.github/workflows/refresh_downstream.yml). Useragents.io provides a private API for this module which exposes the last 7 days of newly seen user agents for the currently tracked browsers. If a new major version of one of the tracked browsers is encountered with a Chromium version that meets or exceeds the previous latest version of that browser, it is added to the [src/data/downstream-browsers.json](src/data/downstream-browsers.json) file with the date it was first seen by useragents.io as its release date.
+
+KaiOS is an exception - its upstream version mappings are handled separately from the other browsers because they happen very infrequently.
+
+### List of downstream browsers
+
+| Browser               | ID                        | Core    | Source                    |
+| --------------------- | ------------------------- | ------- | ------------------------- |
+| Chrome                | `chrome`                  | `true`  | MDN `browser-compat-data` |
+| Chrome for Android    | `chrome_android`          | `true`  | MDN `browser-compat-data` |
+| Edge                  | `edge`                    | `true`  | MDN `browser-compat-data` |
+| Firefox               | `firefox`                 | `true`  | MDN `browser-compat-data` |
+| Firefox for Android   | `firefox_android`         | `true`  | MDN `browser-compat-data` |
+| Safari                | `safari`                  | `true`  | MDN `browser-compat-data` |
+| Safari on iOS         | `safari_ios`              | `true`  | MDN `browser-compat-data` |
+| Opera                 | `opera`                   | `false` | MDN `browser-compat-data` |
+| Opera Android         | `opera_android`           | `false` | MDN `browser-compat-data` |
+| Samsung Internet      | `samsunginternet_android` | `false` | MDN `browser-compat-data` |
+| WebView Android       | `webview_android`         | `false` | MDN `browser-compat-data` |
+| QQ Browser Mobile     | `qq_android`              | `false` | useragents.io             |
+| UC Browser Mobile     | `uc_android`              | `false` | useragents.io             |
+| Yandex Browser Mobile | `ya_android`              | `false` | useragents.io             |
+| KaiOS                 | `kai_os`                  | `false` | Manual                    |
+| Facebook for Android  | `facebook_android`        | `false` | useragents.io             |
+| Instagram for Android | `instagram_android`       | `false` | useragents.io             |
+
+> [!NOTE]
+> All the non-core browsers currently included implement Chromium or Gecko. Their inclusion in any of the above methods is based on the Baseline feature set supported by the Chromium or Gecko version they implement, not their release date.

node_modules/baseline-browser-mapping/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "baseline-browser-mapping",
+  "main": "./dist/index.cjs",
+  "version": "2.9.19",
+  "description": "A library for obtaining browser versions with their maximum supported Baseline feature set and Widely Available status.",
+  "exports": {
+    ".": {
+      "require": "./dist/index.cjs",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./legacy": {
+      "require": "./dist/index.cjs",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "jsdelivr": "./dist/index.js",
+  "files": [
+    "dist/*",
+    "!dist/scripts/*",
+    "LICENSE.txt",
+    "README.md"
+  ],
+  "types": "./dist/index.d.ts",
+  "type": "module",
+  "bin": {
+    "baseline-browser-mapping": "dist/cli.js"
+  },
+  "scripts": {
+    "fix-cli-permissions": "output=$(npx baseline-browser-mapping 2>&1); path=$(printf '%s\n' \"$output\" | sed -n 's/^.*: \\(.*\\): Permission denied$/\\1/p; t; s/^\\(.*\\): Permission denied$/\\1/p'); if [ -n \"$path\" ]; then echo \"Permission denied for: $path\"; echo \"Removing $path ...\"; rm -rf \"$path\"; else echo \"$output\"; fi",
+    "test:format": "npx prettier --check .",
+    "test:lint": "npx eslint .",
+    "test:jasmine": "npx jasmine",
+    "test:jasmine-browser": "npx jasmine-browser-runner runSpecs --config ./spec/support/jasmine-browser.js",
+    "test": "npm run build && npm run fix-cli-permissions && npm run test:format && npm run test:lint && npm run test:jasmine && npm run test:jasmine-browser",
+    "build": "rm -rf dist; npx prettier . --write; rollup -c; rm -rf ./dist/scripts/expose-data.d.ts ./dist/cli.d.ts",
+    "refresh-downstream": "npx tsx scripts/refresh-downstream.ts",
+    "refresh-static": "npx tsx scripts/refresh-static.ts",
+    "update-data-file": "npx tsx scripts/update-data-file.ts; npx prettier ./src/data/data.js --write",
+    "update-data-dependencies": "npm i @mdn/browser-compat-data@latest web-features@latest -D",
+    "check-data-changes": "git diff --name-only | grep -q '^src/data/data.js$' && echo 'changes-available=TRUE' || echo 'changes-available=FALSE'"
+  },
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@mdn/browser-compat-data": "^7.2.5",
+    "@rollup/plugin-terser": "^0.4.4",
+    "@rollup/plugin-typescript": "^12.1.3",
+    "@types/node": "^22.15.17",
+    "eslint-plugin-new-with-error": "^5.0.0",
+    "jasmine": "^5.8.0",
+    "jasmine-browser-runner": "^3.0.0",
+    "jasmine-spec-reporter": "^7.0.0",
+    "prettier": "^3.5.3",
+    "rollup": "^4.44.0",
+    "tslib": "^2.8.1",
+    "typescript": "^5.7.2",
+    "typescript-eslint": "^8.35.0",
+    "web-features": "^3.14.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/web-platform-dx/baseline-browser-mapping.git"
+  }
+}

package-lock.json

@@ -0,0 +1,22 @@
+{
+  "name": "shelfmark",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "devDependencies": {
+        "baseline-browser-mapping": "^2.9.19"
+      }
+    },
+    "node_modules/baseline-browser-mapping": {
+      "version": "2.9.19",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz",
+      "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "baseline-browser-mapping": "dist/cli.js"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "baseline-browser-mapping": "^2.9.19"
+  }
+}

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "shelfmark"
+version = "0.1.0"
+description = "Shelfmark - Book Downloader"
+requires-python = ">=3.10"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--tb=short",
+]
+markers = [
+    "integration: marks tests that require running services (deselect with '-m \"not integration\"')",
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks end-to-end tests that require the full application stack",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+ignore_missing_imports = true

readme.md

@@ -1,345 +1,272 @@
-# 📚 Calibre-Web-Automated-Book-Downloader
+# 📚 Shelfmark: Book Downloader
 
-![Calibre-Web Automated Book Downloader](src/frontend/public/logo.png 'Calibre-Web Automated Book Downloader')
+Formerly *Calibre Web Automated Book Downloader (CWABD)*
 
-An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
+<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
+
+Shelfmark is a self-hosted web interface for searching and downloading books and audiobooks from multiple sources. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
+
+**Fully standalone** - no external dependencies required. Works great alongside the following library tools, with support for automatic imports:
+- [Calibre](https://calibre-ebook.com/)
+- [Calibre-Web](https://github.com/janeczku/calibre-web)
+- [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated)
+- [Booklore](https://github.com/booklore-app/booklore)
+- [Audiobookshelf](https://github.com/advplyr/audiobookshelf)
 
 ## ✨ Features
 
-- 🌐 User-friendly web interface for book search and download
-- 🔄 Automated download to your specified ingest folder
-- 🔌 Seamless integration with Calibre-Web-Automated
-- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr)
-- 🛡️ Cloudflare bypass capability for reliable downloads
-- 🐳 Docker-based deployment for quick setup
+- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
+- **Multiple Sources** - Popular archive websites, Torrent, Usenet, and IRC download support
+- **Audiobook Support** - Full audiobook search and download with dedicated processing
+- **Two Search Modes**:
+  - **Direct** - Search popular web sources
+  - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
+- **Multi-User & Requests** - Share your instance with others, let users browse and request books, and manage approvals with configurable notifications
+- **Authentication** - Built-in login, OIDC single sign-on, proxy auth, and Calibre-Web database support
+- **Real-Time Progress** - Unified download queue with live status updates across all sources
+- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
 
 ## 🖼️ Screenshots
 
-![Main search interface Screenshot](README_images/search.png 'Main search interface')
+**Home screen**
+![Home screen](README_images/homescreen.png 'Home screen')
 
-![Details modal Screenshot placeholder](README_images/details.png 'Details modal')
+**Search results**
+![Search results](README_images/search-results.png 'Search results')
 
-![Download queue Screenshot placeholder](README_images/downloading.png 'Download queue')
+**Multi-source downloads**
+![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
+
+**Download queue**
+![Download queue](README_images/downloads.png 'Download queue')
 
 ## 🚀 Quick Start
 
 ### Prerequisites
 
-- Docker
-- Docker Compose
-- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended)
+- Docker & Docker Compose
 
-### Installation Steps
-
-1. Get the docker-compose.yml:
+### Installation
 
+1. Download the [docker-compose file](compose/docker-compose.yml):
    ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml
+   curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
    ```
 
 2. Start the service:
-
    ```bash
    docker compose up -d
    ```
 
-3. Access the web interface at `http://localhost:8084`
+3. Open `http://localhost:8084`
 
-## ⚙️ Configuration
+That's it! Configure settings through the web interface as needed.
 
-### Environment Variables
-
-#### Application Settings
-
-| Variable          | Description             | Default Value      |
-| ----------------- | ----------------------- | ------------------ |
-| `FLASK_PORT`      | Web interface port      | `8084`             |
-| `FLASK_HOST`      | Web interface binding   | `0.0.0.0`          |
-| `DEBUG`           | Debug mode toggle       | `false`            |
-| `INGEST_DIR`      | Book download directory | `/cwa-book-ingest` |
-| `TZ`              | Container timezone      | `UTC`              |
-| `UID`             | Runtime user ID         | `1000`             |
-| `GID`             | Runtime group ID        | `100`              |
-| `CWA_DB_PATH`     | Calibre-Web's database  | None               |
-| `ENABLE_LOGGING`  | Enable log file         | `true`             |
-| `LOG_LEVEL`       | Log level to use        | `info`             |
-| `SESSION_COOKIE_SECURE` | Secure cookie enforcement - Use for HTTPS connections only | `false` |
-| `CALIBRE_WEB_URL` | Custom WebUI library link | None |
-
-If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
-
-Set `CALIBRE_WEB_URL` to your Calibre-Web / Booklore base URL. A ‘Go to library’ button will appear in the Web UI for quick access while downloading, and it also provides library access when CWA-BD is installed as a mobile PWA.
-
-If logging is enabled, log folder default location is `/var/log/cwa-book-downloader`
-Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages.
-
-Note that if using TOR, the TZ will be calculated automatically based on IP.
-
-#### Download Settings
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `MAX_RETRY`            | Maximum retry attempts                                    | `3`                               |
-| `DEFAULT_SLEEP`        | Retry delay (seconds)                                     | `5`                               |
-| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds)                           | `5`                               |
-| `SUPPORTED_FORMATS`    | Supported book formats                                    | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
-| `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
-| `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
-| `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
-| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
-| `ALLOW_USE_WELIB`       | Allow usage of welib for downloading books if found there | `true`                            |
-
-If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
-
-Use the following environment variables to set specific folders in which to download 
-different content types (Book, Magazine, Comic, etc.):
-
-| Variable                        | Description                    | Default Value |
-|---------------------------------|--------------------------------|---------------|
-| `INGEST_DIR_BOOK_FICTION`       | Book (fiction) folder name     | ``            |
-| `INGEST_DIR_BOOK_NON_FICTION`   | Book (non-fiction) folder name | ``            |
-| `INGEST_DIR_BOOK_UNKNOWN`       | Book (unknown) folder name     | ``            |
-| `INGEST_DIR_MAGAZINE`           | Magazine folder name           | ``            |
-| `INGEST_DIR_COMIC_BOOK`         | Comic book folder name         | ``            |
-| `INGEST_DIR_AUDIOBOOK`          | Audiobook folder name          | ``            |
-| `INGEST_DIR_STANDARDS_DOCUMENT` | Standards document folder name | ``            |
-| `INGEST_DIR_MUSICAL_SCORE`      | Musical score folder name      | ``            |
-
-If no specific path is set for a content type the default is `INGEST_DIR`.  
-Remember to map the specified paths to where your instance of Calibre-Web-Automated (CWA) will find them, e.g.:  
-```
-volumes:
-    - /tmp/data/calibre-web/comicbook-ingest:/cwa-comicbook-ingest
-```
-if `INGEST_DIR_COMIC_BOOK=/cwa-comicbook-ingest` and your CWA is configured to use `/tmp/data/calibre-web/comicbook-ingest` 
-for comic books.
-
-
-#### AA 
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
-
-If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
-If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.
-
-#### Network Settings
-
-| Variable               | Description                     | Default Value           |
-| ---------------------- | ------------------------------- | ----------------------- |
-| `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
-| `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
-| `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | Custom DNS IP                   | ``                      |
-| `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
-
-For proxy configuration, you can specify URLs in the following format:
-```bash
-# Basic proxy
-HTTP_PROXY=http://proxy.example.com:8080
-HTTPS_PROXY=http://proxy.example.com:8080
-
-# Proxy with authentication
-HTTP_PROXY=http://username:password@proxy.example.com:8080
-HTTPS_PROXY=http://username:password@proxy.example.com:8080
-```
-
-
-The `CUSTOM_DNS` setting supports two formats:
-
-1. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
-   - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses in the same string
-
-2. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS
-   - `quad9` - Quad9 DNS
-   - `cloudflare` - Cloudflare DNS
-   - `opendns` - OpenDNS
-
-For users experiencing ISP-level website blocks (such as Virgin Media in the UK), using alternative DNS providers like Cloudflare may help bypass these restrictions
-
-If a `CUSTOM_DNS` is specified from the preset providers, you can also set a `USE_DOH=true` to force using DNS over HTTPS,
-which might also help in certain network situations. Note that only `google`, `quad9`, `cloudflare` and `opendns` are 
-supported for now, and any other value in `CUSTOM_DNS` will make the `USE_DOH` flag ignored.
-
-Try something like this :
-```bash
-CUSTOM_DNS=cloudflare
-USE_DOH=true
-```
-
-#### Custom configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `CUSTOM_SCRIPT`        | Path to an executable script that tuns after each download  | ``                      |
-
-If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation.
-
-The script is called with the full path of the downloaded file as its argument. Important notes:
-- The script must preserve the original filename for proper processing
-- The file can be modified or even deleted if needed
-- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted)
-
-You can specify these configuration in this format :
-```
-environment:
-  - CUSTOM_SCRIPT=/scripts/process-book.sh
-
-volumes:
-  - local/scripts/custom_script.sh:/scripts/process-book.sh
-```
-
-### Volume Configuration
+### Volume Setup
 
 ```yaml
 volumes:
-  - /your/local/path:/cwa-book-ingest
-  - /cwa/config/path/app.db:/auth/app.db:ro
+  - /your/config/path:/config # Config, database, and artwork cache directory
+  - /your/download/path:/books # Downloaded books
+  - /client/path:/client/path # Optional: For Torrent/Usenet downloads, match your client directory exactly. 
 ```
-**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777
 
-Mount should align with your Calibre-Web-Automated ingest folder.
+> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
 
-## Variants:
+> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
 
-### 🧅 Tor Variant
+## ⚙️ Configuration
 
-This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions.
+### Search Modes
 
-To use the Tor variant:
+**Direct** (default)
+- Works out of the box, no setup required
+- Searches a huge library of books directly
+- Returns downloadable releases immediately
 
-1.  Get the Tor-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.tor.yml up -d
-    ```
+**Universal**
+- Cleaner search results via metadata providers (Hardcover is recommended)
+- Aggregates releases from multiple configured sources
+- Full Audiobook support
+- Requires manual setup (API keys, additional sources)
 
-**Important Considerations for Tor:**
+### Environment Variables
 
-*   **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying.
-*   **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set.
-*   **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor.
+Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
 
-### External Cloudflare resolver variant
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FLASK_PORT` | Web interface port | `8084` |
+| `INGEST_DIR` | Book download directory | `/books` |
+| `TZ` | Container timezone | `UTC` |
+| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` |
+| `SEARCH_MODE` | `direct` or `universal` | `direct` |
+| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
 
-This variant allows the application to use an external service to bypass Cloudflare protection, instead of relying on the built-in bypasser. This is useful if you already have a dedicated Cloudflare resolver (such as [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) or compatible services like [ByParr](https://github.com/ThePhaseless/Byparr)) running elsewhere.
+See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
 
-#### How it works:
+Some of the additional options available in Settings:
+- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
+- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
+- **AudiobookBay** - Web scraping source for audiobook torrents (audiobooks only)
+- **IRC** - Add details for IRC book sources and download directly from the UI
+- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
+- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
+- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
+- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
+- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
 
-- When enabled, all requests that require Cloudflare bypass are sent to your external resolver service.
-- The application communicates with the resolver using its API.
-- This approach can improve reliability and performance, especially if your external resolver is optimized or shared across multiple applications.
+## 🐳 Docker Variants
 
-#### Configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `EXT_BYPASSER_URL`     | The full URL of your external resolver (required)           |                         |
-| `EXT_BYPASSER_PATH`    | API path for the resolver (usually `/v1`)                   | `/v1`                   |
-| `EXT_BYPASSER_TIMEOUT` | Timeout for page loading (in milliseconds)                  | `60000`                 |
-
-#### Important
-
-This feature follows the same configuration of the built-in Cloudflare bypasser, so you should turn on the `USE_CF_BYPASS` configuration to enable it.
-
-#### To use the External Cloudflare resolver variant:
-
-1.  Get the extbp-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.extbp.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.extbp.yml up -d
-    ```
-
-#### Compatibility:
-This feature is designed to work with any resolver that implements the `FlareSolverr` API schema, including `ByParr` and similar projects.
-
-#### Benefits:
-
-- Centralizes Cloudflare bypass logic for easier maintenance.
-- Can leverage more powerful or distributed resolver infrastructure.
-- Reduces load on the main application container.
-
-## 🏗️ Architecture
-
-The application consists of a Flask backend with a React-based frontend:
-
-### Backend
-- **Flask Application**: Python-based backend (`app.py`, `backend.py`) providing REST API and WebSocket support
-- **Download Manager**: Handles book search, download requests, and queue management (`downloader.py`, `book_manager.py`)
-- **Network Layer**: Cloudflare bypass and proxy support (`cloudflare_bypasser.py`, `network.py`)
-
-### Frontend
-- **React + TypeScript**: Modern web interface built with Vite (`src/frontend`)
-- **Real-time Updates**: WebSocket integration for live download status
-- **Responsive UI**: TailwindCSS-based design for mobile and desktop
-
-For frontend development, use the provided Makefile:
+### Standard
 ```bash
-make install  # Install dependencies
-make dev      # Start development server
-make build    # Build for production
-```
-If you run the docker compose file, the frontend will be built and served automatically. But if you run the frontend dev server it will supercede the docker compose frontend.
-
-## 🏥 Health Monitoring
-
-Built-in health checks monitor:
-
-- Web interface availability
-- Download service status
-- Cloudflare bypass service connection
-
-Checks run every 30 seconds with a 30-second timeout and 3 retries.
-You can enable by adding this to your compose :
-```
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD curl -s http://localhost:8084/api/status || exit 1
+docker compose up -d
 ```
 
-## 📝 Logging
+The full-featured image with built-in Cloudflare bypass.
 
-Logs are available in:
+#### Enable Tor Routing
+Routes all traffic through Tor for enhanced privacy:
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
+docker compose -f docker-compose.tor.yml up -d
+```
 
-- Container: `/var/logs/cwa-book-downloader.log`
-- Docker logs: Access via `docker logs`
+**Notes:**
+- Requires `NET_ADMIN` and `NET_RAW` capabilities
+- Timezone is auto-detected from Tor exit node
+- Custom DNS/proxy settings are ignored when Tor is active
 
-## 🤝 Contributing
+### Lite
+A smaller image without the built-in Cloudflare bypasser. Ideal for:
 
-Contributions are welcome! Feel free to submit a Pull Request.
+- **External bypassers** - Already running FlareSolverr or ByParr for other services
+- **Fast downloads** - Using fast download sources
+- **Alternative sources only** - Exclusively using Prowlarr, AudiobookBay, IRC, or other sources
+- **Audiobooks** - Using Shelfmark exclusively for audiobooks
 
-## 📄 License
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
+docker compose -f docker-compose.lite.yml up -d
+```
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
 
-## ⚠️ Important Disclaimers
+## 🔐 Authentication
+
+Authentication is optional but recommended for shared or exposed instances. Multiple authentication methods are available in Settings:
+
+**1. Single Username/Password**
+
+**2. Proxy (Forward) Authentication**
+
+Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
+
+**3. OIDC (OpenID Connect)**
+
+Integrate with your identity provider (Authelia, Authentik, Keycloak, etc.) for single sign-on. Supports PKCE flow, auto-discovery, group-based admin mapping, and auto-provisioning of new users.
+
+**4. Calibre-Web Database**
+
+If you're running Calibre-Web, you can reuse its user database by mounting it:
+
+```yaml
+volumes:
+  - /path/to/calibre-web/app.db:/auth/app.db:ro
+```
+
+### Multi-User Support
+
+With any authentication method enabled, Shelfmark supports multi-user management with admin/user roles. Users can have per-user settings for download destinations, email recipients, and notification preferences. Non-admin users only see their own downloads and can submit book requests for admin review. Admins can configure request policies per source to control whether users can download directly, must submit a request, or are blocked entirely.
+
+## Health Monitoring
+
+The application exposes a health endpoint at `/api/health` (no authentication required). Add a health check to your compose:
+
+```yaml
+healthcheck:
+  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/health"]
+  interval: 30s
+  timeout: 30s
+  retries: 3
+```
+
+## Logging
+
+Logs are available via:
+- `docker logs <container-name>`
+- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
+
+Log level is configurable via Settings or `LOG_LEVEL` environment variable.
+
+## Development
+
+```bash
+# Frontend development
+make install     # Install dependencies
+make dev         # Start Vite dev server (localhost:5173)
+make build       # Production build
+make typecheck   # TypeScript checks
+
+# Backend (Docker)
+make up          # Start backend via docker-compose.dev.yml
+make down        # Stop services
+make refresh     # Rebuild and restart
+make restart     # Restart container
+```
+
+The frontend dev server proxies to the backend on port 8084.
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      Web Interface                          │
+│                 (React + TypeScript + Vite)                 │
+├─────────────────────────────────────────────────────────────┤
+│                      Flask Backend                          │
+│                   (REST API + WebSocket)                    │
+├───────────────────┬─────────────────────┬───────────────────┤
+│ Metadata Providers│   Download Queue    │  Cloudflare       │
+│                   │   & Orchestrator    │  Bypass           │
+├───────────────────┼─────────────────────┼───────────────────┤
+│ • Hardcover       │ • Task scheduling   │ • Internal        │
+│ • Open Library    │ • Progress tracking │ • External        │
+│                   │ • Retry logic       │   (FlareSolverr)  │
+├───────────────────┴─────────────────────┴───────────────────┤
+│                     Release Sources                         │
+├─────────────────────────────────────────────────────────────┤
+│ • Direct Download (Web Sources → Mirrors → Fallbacks)       │
+├─────────────────────────────────────────────────────────────┤
+│                     Network Layer                           │
+├─────────────────────────────────────────────────────────────┤
+│ • Auto DNS rotation  • Mirror failover  • Resume support    │
+└─────────────────────────────────────────────────────────────┘
+```
+
+The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
+
+## Contributing
+
+Shelfmark's core feature set is now largely complete. Development going forward will focus on stability, bug fixes, and maintenance rather than major new features. Contributions in these areas are welcome - please file issues or submit pull requests on GitHub.
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## ⚠️ Disclaimers
 
 ### Copyright Notice
 
-While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for:
-
+This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
 - Ensuring they have the right to download requested materials
 - Respecting copyright laws and intellectual property rights
 - Using the tool in compliance with their local regulations
 
-### Duplicate Downloads Warning
+### Library Integration
 
-Please note that the current version:
+Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
 
-- Does not check for existing files in the download directory
-- Does not verify if books already exist in your Calibre database
-- Exercise caution when requesting multiple books to avoid duplicates
-
-## 💬 Support
-
-For issues or questions, please file an issue on the GitHub repository.
+## Support
 
+For issues or questions, please [file an issue](https://github.com/calibrain/shelfmark/issues) on GitHub.

requirements-base.txt

@@ -1,4 +1,4 @@
-flask
+flask>=3.1.0,<3.1.3  # Temporary: Flask 3.1.3 breaks flask-socketio (github.com/miguelgrinberg/Flask-SocketIO/pull/2153)
 flask-cors
 flask-socketio
 python-socketio
@@ -11,3 +11,8 @@ gevent
 gevent-websocket
 psutil
 emoji
+rarfile
+qbittorrent-api
+transmission-rpc
+authlib>=1.6.6,<1.7
+apprise>=1.9.0

requirements-shelfmark.txt

@@ -1,4 +1,4 @@
 pyvirtualdisplay
 pyautogui
-seleniumbase>=4.41.1
+seleniumbase==4.45.10
 python-xlib

scripts/fix_permissions.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fix permissions on all configured directories.
+
+This script is called by the entrypoint to ensure all user-configured
+directories have correct ownership. It reads directory paths from:
+- CONFIG_DIR environment variable
+- Config files in CONFIG_DIR/plugins/
+
+Outputs directory paths that need permission fixing (one per line).
+The entrypoint handles the actual chown operations.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def get_directories_from_config() -> set[str]:
+    """Extract all directory paths from config files."""
+    directories = set()
+
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    plugins_dir = config_dir / "plugins"
+
+    if not plugins_dir.exists():
+        return directories
+
+    # Keys that contain directory paths
+    directory_keys = {
+        # Main destinations
+        "DESTINATION",
+        "DESTINATION_AUDIOBOOK",
+        # Content type routing directories
+        "AA_CONTENT_TYPE_DIR_FICTION",
+        "AA_CONTENT_TYPE_DIR_NON_FICTION",
+        "AA_CONTENT_TYPE_DIR_UNKNOWN",
+        "AA_CONTENT_TYPE_DIR_MAGAZINE",
+        "AA_CONTENT_TYPE_DIR_COMIC",
+        "AA_CONTENT_TYPE_DIR_STANDARDS",
+        "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+        "AA_CONTENT_TYPE_DIR_OTHER",
+        # Legacy keys (in case of old configs)
+        "INGEST_DIR",
+        "INGEST_DIR_AUDIOBOOK",
+        "INGEST_DIR_BOOK_FICTION",
+        "INGEST_DIR_BOOK_NON_FICTION",
+        "INGEST_DIR_BOOK_UNKNOWN",
+        "INGEST_DIR_MAGAZINE",
+        "INGEST_DIR_COMIC_BOOK",
+        "INGEST_DIR_STANDARDS_DOCUMENT",
+        "INGEST_DIR_MUSICAL_SCORE",
+        "INGEST_DIR_OTHER",
+        "LIBRARY_PATH",
+        "LIBRARY_PATH_AUDIOBOOK",
+    }
+
+    # Read all JSON config files
+    for config_file in plugins_dir.glob("*.json"):
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+
+            for key in directory_keys:
+                if key in config:
+                    value = config[key]
+                    if value and isinstance(value, str) and value.startswith("/"):
+                        directories.add(value)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    return directories
+
+
+def main():
+    """Output all configured directories that exist."""
+    directories = get_directories_from_config()
+
+    # Filter to directories that actually exist
+    existing = []
+    for dir_path in directories:
+        path = Path(dir_path)
+        if path.exists() and path.is_dir():
+            existing.append(dir_path)
+
+    # Output one directory per line
+    for dir_path in sorted(existing):
+        print(dir_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_env_docs.py

@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""Generate markdown documentation for environment variables from the settings registry.
+
+This script extracts all settings that support environment variable configuration
+and generates a comprehensive markdown file documenting each option.
+
+Usage:
+    python scripts/generate_env_docs.py [--output path/to/output.md]
+
+The generated documentation includes:
+- Environment variable name
+- Description
+- Type (string, number, boolean, etc.)
+- Default value
+- Organizational grouping by settings tab/group
+"""
+
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Add project root to path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+
+def get_field_type_name(field) -> str:
+    """Get a human-readable type name for a field."""
+    from shelfmark.core.settings_registry import (
+        CheckboxField,
+        MultiSelectField,
+        NumberField,
+        OrderableListField,
+        PasswordField,
+        SelectField,
+        TextField,
+    )
+
+    if isinstance(field, CheckboxField):
+        return "boolean"
+    elif isinstance(field, NumberField):
+        return "number"
+    elif isinstance(field, SelectField):
+        return "string (choice)"
+    elif isinstance(field, MultiSelectField):
+        return "string (comma-separated)"
+    elif isinstance(field, OrderableListField):
+        return "JSON array"
+    elif isinstance(field, PasswordField):
+        return "string (secret)"
+    elif isinstance(field, TextField):
+        return "string"
+    else:
+        return "string"
+
+
+def format_default_value(field) -> str:
+    """Format the default value for display."""
+    default = field.default
+
+    if default is None:
+        return "_none_"
+    elif isinstance(default, bool):
+        return f"`{str(default).lower()}`"
+    elif isinstance(default, (int, float)):
+        return f"`{default}`"
+    elif isinstance(default, str):
+        if default == "":
+            return "_empty string_"
+        return f"`{default}`"
+    elif isinstance(default, list):
+        if not default:
+            return "_empty list_"
+        # For simple lists, show comma-separated values
+        if all(isinstance(item, str) for item in default):
+            return f"`{','.join(default)}`"
+        # For complex lists (e.g., OrderableListField defaults), summarize
+        return f"_see UI for defaults_"
+    else:
+        return f"`{default}`"
+
+
+def get_select_options(field) -> Optional[List[str]]:
+    """Get the available options for a SelectField.
+
+    Returns options formatted as 'value (label)' or just 'value' if they match,
+    so users know the actual values to use in environment variables.
+    """
+    from shelfmark.core.settings_registry import SelectField
+
+    if not isinstance(field, SelectField):
+        return None
+
+    options = field.options
+    if callable(options):
+        try:
+            options = options()
+        except Exception:
+            return None
+
+    if not options:
+        return None
+
+    result = []
+    for opt in options:
+        value = opt.get("value", "")
+        label = opt.get("label", "")
+
+        # Format as "value (label)" unless they're the same or value is empty
+        if value == "":
+            result.append(f'`""` ({label})')
+        elif value == label or not label:
+            result.append(f"`{value}`")
+        else:
+            result.append(f"`{value}` ({label})")
+
+    return result
+
+
+def _generate_bootstrap_env_docs() -> List[str]:
+    """Generate documentation for bootstrap environment variables from env.py."""
+    # These are environment variables defined in env.py that are used before
+    # the settings registry is available
+    bootstrap_vars = [
+        {
+            "name": "CONFIG_DIR",
+            "description": "Directory for storing configuration files and plugin settings.",
+            "type": "string (path)",
+            "default": "/config",
+        },
+        {
+            "name": "LOG_ROOT",
+            "description": "Root directory for log files.",
+            "type": "string (path)",
+            "default": "/var/log/",
+        },
+        {
+            "name": "TMP_DIR",
+            "description": "Staging directory for downloads before moving to destination.",
+            "type": "string (path)",
+            "default": "/tmp/shelfmark",
+        },
+        {
+            "name": "ENABLE_LOGGING",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
+            "type": "boolean",
+            "default": "true",
+        },
+        {
+            "name": "FLASK_HOST",
+            "description": "Host address for the Flask web server.",
+            "type": "string",
+            "default": "0.0.0.0",
+        },
+        {
+            "name": "FLASK_PORT",
+            "description": "Port number for the Flask web server.",
+            "type": "number",
+            "default": "8084",
+        },
+        {
+            "name": "SESSION_COOKIE_SECURE",
+            "description": "Enable secure cookies (requires HTTPS).",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "CWA_DB_PATH",
+            "description": "Path to the Calibre-Web database for authentication integration.",
+            "type": "string (path)",
+            "default": "/auth/app.db",
+        },
+        {
+            "name": "DOCKERMODE",
+            "description": "Indicates the application is running inside a Docker container.",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "ONBOARDING",
+            "description": "Show the onboarding wizard on first run. Set to false to skip (useful for ephemeral storage).",
+            "type": "boolean",
+            "default": "true",
+        },
+    ]
+
+    lines = [
+        "## Bootstrap Configuration",
+        "",
+        "These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
+        "",
+        "| Variable | Description | Type | Default |",
+        "|----------|-------------|------|---------|",
+    ]
+
+    for var in bootstrap_vars:
+        lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
+
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for var in bootstrap_vars:
+        lines.append(f"#### `{var['name']}`")
+        lines.append("")
+        lines.append(var["description"])
+        lines.append("")
+        lines.append(f"- **Type:** {var['type']}")
+        lines.append(f"- **Default:** `{var['default']}`")
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def generate_env_docs() -> str:
+    """Generate markdown documentation for all environment variables."""
+    # Import settings modules to ensure all settings are registered
+    import shelfmark.config.settings  # noqa: F401
+    import shelfmark.release_sources.irc.settings  # noqa: F401
+    import shelfmark.release_sources.prowlarr.settings  # noqa: F401
+    import shelfmark.metadata_providers.hardcover  # noqa: F401
+    import shelfmark.metadata_providers.openlibrary  # noqa: F401
+    import shelfmark.metadata_providers.googlebooks  # noqa: F401
+
+    from shelfmark.core.settings_registry import (
+        ActionButton,
+        HeadingField,
+        get_all_groups,
+        get_all_settings_tabs,
+    )
+
+    tabs = get_all_settings_tabs()
+    groups = {g.name: g for g in get_all_groups()}
+
+    # Organize tabs by group
+    grouped_tabs: Dict[Optional[str], List] = {None: []}
+    for group_name in groups:
+        grouped_tabs[group_name] = []
+
+    for tab in tabs:
+        group_name = tab.group
+        if group_name not in grouped_tabs:
+            grouped_tabs[group_name] = []
+        grouped_tabs[group_name].append(tab)
+
+    # Build markdown output
+    lines = [
+        "# Environment Variables",
+        "",
+        "This document lists all configuration options that can be set via environment variables.",
+        "",
+        "> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
+        "",
+        "## Table of Contents",
+        "",
+    ]
+
+    # Generate TOC
+    toc_entries = [
+        "- [Bootstrap Configuration](#bootstrap-configuration)",
+    ]
+
+    # Ungrouped tabs first
+    for tab in grouped_tabs.get(None, []):
+        anchor = tab.display_name.lower().replace(" ", "-")
+        toc_entries.append(f"- [{tab.display_name}](#{anchor})")
+
+    # Then grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if group_tabs:
+            anchor = group.display_name.lower().replace(" ", "-")
+            toc_entries.append(f"- [{group.display_name}](#{anchor})")
+            for tab in group_tabs:
+                sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
+                toc_entries.append(f"  - [{tab.display_name}](#{sub_anchor})")
+
+    lines.extend(toc_entries)
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Add bootstrap environment variables documentation
+    lines.extend(_generate_bootstrap_env_docs())
+
+    # Generate documentation for ungrouped tabs
+    for tab in grouped_tabs.get(None, []):
+        lines.extend(_generate_tab_docs(tab))
+
+    # Generate documentation for grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if not group_tabs:
+            continue
+
+        lines.append(f"## {group.display_name}")
+        lines.append("")
+
+        for tab in group_tabs:
+            lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
+
+    return "\n".join(lines)
+
+
+def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
+    """Generate documentation for a single settings tab."""
+    from shelfmark.core.settings_registry import ActionButton, HeadingField
+
+    lines = []
+
+    # Section header
+    if group_prefix:
+        lines.append(f"### {group_prefix}: {tab.display_name}")
+        anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
+    else:
+        lines.append(f"## {tab.display_name}")
+
+    lines.append("")
+
+    # Collect env-supported fields
+    env_fields = []
+    for field in tab.fields:
+        # Skip non-value fields
+        if isinstance(field, (ActionButton, HeadingField)):
+            continue
+
+        # Skip fields that don't support ENV vars
+        if not getattr(field, "env_supported", True):
+            continue
+
+        env_fields.append(field)
+
+    if not env_fields:
+        lines.append("_No environment variables for this section._")
+        lines.append("")
+        return lines
+
+    # Generate table
+    lines.append("| Variable | Description | Type | Default |")
+    lines.append("|----------|-------------|------|---------|")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        description = field.description or field.label
+        # Clean up description for table (remove newlines, escape pipes)
+        description = description.replace("\n", " ").replace("|", "\\|").strip()
+
+        field_type = get_field_type_name(field)
+        default = format_default_value(field)
+
+        lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
+
+    lines.append("")
+
+    # Add detailed documentation for each field
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        lines.append(f"#### `{env_var}`")
+        lines.append("")
+        lines.append(f"**{field.label}**")
+        lines.append("")
+
+        if field.description:
+            lines.append(field.description)
+            lines.append("")
+
+        lines.append(f"- **Type:** {get_field_type_name(field)}")
+        lines.append(f"- **Default:** {format_default_value(field)}")
+
+        if getattr(field, "required", False):
+            lines.append("- **Required:** Yes")
+
+        if getattr(field, "requires_restart", False):
+            lines.append("- **Requires restart:** Yes")
+
+        # Show options for SelectField
+        options = get_select_options(field)
+        if options:
+            lines.append(f"- **Options:** {', '.join(options)}")
+
+        # Show constraints for NumberField
+        from shelfmark.core.settings_registry import NumberField
+        if isinstance(field, NumberField):
+            constraints = []
+            if field.min_value is not None:
+                constraints.append(f"min: {field.min_value}")
+            if field.max_value is not None:
+                constraints.append(f"max: {field.max_value}")
+            if constraints:
+                lines.append(f"- **Constraints:** {', '.join(constraints)}")
+
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate markdown documentation for environment variables"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=project_root / "docs" / "environment-variables.md",
+        help="Output file path (default: docs/environment-variables.md)",
+    )
+    parser.add_argument(
+        "--stdout",
+        action="store_true",
+        help="Print to stdout instead of file",
+    )
+    args = parser.parse_args()
+
+    docs = generate_env_docs()
+
+    if args.stdout:
+        print(docs)
+    else:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(docs)
+        print(f"Generated: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_clients.py

@@ -0,0 +1,555 @@
+#!/usr/bin/env python3
+"""
+Test script for download client implementations.
+
+Usage:
+    1. Start the test stack:
+       docker compose -f docker-compose.test-clients.yml up -d
+
+    2. Wait for containers to initialize (first run takes ~30s)
+
+    3. Run this script to verify clients are accessible:
+       python scripts/test_clients.py
+
+    4. Access cwabd at http://localhost:8084
+       - Go to Settings > Prowlarr > Download Clients
+       - Select a client from the dropdown
+       - Click "Test Connection" to verify
+
+Web UIs:
+    - cwabd:        http://localhost:8084
+    - qBittorrent:  http://localhost:8080
+    - Transmission: http://localhost:9091
+    - Deluge:       http://localhost:8112
+    - NZBGet:       http://localhost:6789
+    - SABnzbd:      http://localhost:8085
+    - rTorrent:     http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
+
+Prerequisites (for running this script locally):
+    pip install requests transmission-rpc qbittorrent-api
+
+First-Time Setup:
+    qBittorrent:
+        - Check container logs for temporary password: docker logs test-qbittorrent
+        - Login at http://localhost:8080, change password to something known
+        - Default username is 'admin'
+
+    Transmission:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    Deluge:
+        - Access Web UI at http://localhost:8112 (default password: deluge)
+
+    NZBGet:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    SABnzbd:
+        - Complete the setup wizard at http://localhost:8085
+        - API key will be auto-detected by this script
+        - In cwabd, copy API key from SABnzbd Config > General
+"""
+
+import sys
+import time
+from xmlrpc import client
+
+# Test configuration - matches docker-compose.test-clients.yml
+CONFIG = {
+    # Usenet clients
+    "nzbget": {
+        "url": "http://localhost:6789",
+        "username": "admin",
+        "password": "admin",
+    },
+    "sabnzbd": {
+        "url": "http://localhost:8085",
+        "api_key": None,  # Will be read from config on first run
+    },
+    # Torrent clients
+    "qbittorrent": {
+        "url": "http://localhost:8080",
+        "username": "admin",
+        "password": "5NCngsHXm",  # Temp password from: docker logs test-qbittorrent | grep password
+    },
+    "transmission": {
+        "url": "http://localhost:9091",
+        "username": "admin",
+        "password": "admin",
+    },
+    "deluge": {
+        "url": "http://localhost:8112",
+        "password": "deluge",
+    },
+    "rtorrent": {
+        "url": "http://localhost:8000/RPC2",
+    },
+}
+
+# Test magnet link (Ubuntu ISO - legal, small metadata)
+TEST_MAGNET = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=ubuntu-22.04.3-live-server-amd64.iso"
+
+
+def test_nzbget():
+    """Test NZBGet connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing NZBGet")
+    print("=" * 50)
+
+    url = CONFIG["nzbget"]["url"]
+    username = CONFIG["nzbget"]["username"]
+    password = CONFIG["nzbget"]["password"]
+
+    try:
+        # Test connection via JSON-RPC
+        rpc_url = f"{url}/jsonrpc"
+        response = requests.post(
+            rpc_url,
+            json={"method": "version", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("result", "unknown")
+        print(f"  Connected to NZBGet {version}")
+
+        # Test status
+        response = requests.post(
+            rpc_url,
+            json={"method": "status", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        status = response.json().get("result", {})
+        print(f"  Server state: {'Paused' if status.get('ServerPaused') else 'Running'}")
+        print(f"  Downloads in queue: {status.get('DownloadedSizeMB', 0)} MB downloaded")
+
+        print("  SUCCESS: NZBGet is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to NZBGet")
+        print("  Is the container running? docker ps | grep nzbget")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_sabnzbd():
+    """Test SABnzbd connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing SABnzbd")
+    print("=" * 50)
+
+    url = CONFIG["sabnzbd"]["url"]
+    api_key = CONFIG["sabnzbd"]["api_key"]
+
+    # Try to get API key from config if not set
+    if not api_key:
+        try:
+            import os
+            ini_path = ".local/test-clients/sabnzbd/config/sabnzbd.ini"
+            if os.path.exists(ini_path):
+                with open(ini_path) as f:
+                    for line in f:
+                        if line.startswith("api_key"):
+                            api_key = line.split("=")[1].strip()
+                            print(f"  Found API key in config: {api_key[:8]}...")
+                            break
+        except Exception as e:
+            print(f"  Could not read API key from config: {e}")
+
+    if not api_key:
+        print("  ERROR: No API key configured")
+        print("  Please access http://localhost:8085 and complete initial setup")
+        print("  Then copy the API key from Config > General")
+        return False
+
+    try:
+        # Test connection
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "version", "output": "json"},
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("version", "unknown")
+        print(f"  Connected to SABnzbd {version}")
+
+        # Test queue status
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "queue", "output": "json"},
+            timeout=10,
+        )
+        queue = response.json().get("queue", {})
+        print(f"  Queue status: {queue.get('status', 'unknown')}")
+        print(f"  Items in queue: {len(queue.get('slots', []))}")
+
+        print("  SUCCESS: SABnzbd is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to SABnzbd")
+        print("  Is the container running? docker ps | grep sabnzbd")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_qbittorrent():
+    """Test qBittorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing qBittorrent")
+    print("=" * 50)
+
+    try:
+        import qbittorrentapi
+
+        url = CONFIG["qbittorrent"]["url"]
+        username = CONFIG["qbittorrent"]["username"]
+        password = CONFIG["qbittorrent"]["password"]
+
+        # Parse URL for host/port
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+
+        client = qbittorrentapi.Client(
+            host=parsed.hostname,
+            port=parsed.port or 8080,
+            username=username,
+            password=password,
+        )
+
+        # Test connection
+        client.auth_log_in()
+        version = client.app.version
+        print(f"  Connected to qBittorrent {version}")
+
+        # Get torrent list
+        torrents = client.torrents_info()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        result = client.torrents_add(urls=TEST_MAGNET, is_paused=True)
+        if result == "Ok.":
+            # Wait a moment for it to be added
+            time.sleep(1)
+            torrents = client.torrents_info()
+            if torrents:
+                test_torrent = torrents[-1]  # Most recently added
+                print(f"  Added test torrent: {test_torrent.name[:50]}...")
+                print(f"  Status: {test_torrent.state}")
+
+                # Remove it
+                client.torrents_delete(torrent_hashes=test_torrent.hash, delete_files=True)
+                print("  Removed test torrent")
+        else:
+            print(f"  Add result: {result}")
+
+        print("  SUCCESS: qBittorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: qbittorrent-api not installed")
+        print("  Run: pip install qbittorrent-api")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Forbidden" in str(e) or "401" in str(e):
+            print("\n  Authentication failed. Check password:")
+            print("  1. docker logs test-qbittorrent | grep password")
+            print("  2. Login to http://localhost:8080 and set a known password")
+        return False
+
+
+def test_transmission():
+    """Test Transmission connection."""
+    print("\n" + "=" * 50)
+    print("Testing Transmission")
+    print("=" * 50)
+
+    try:
+        from transmission_rpc import Client
+        from urllib.parse import urlparse
+
+        url = CONFIG["transmission"]["url"]
+        parsed = urlparse(url)
+
+        client = Client(
+            host=parsed.hostname,
+            port=parsed.port or 9091,
+            username=CONFIG["transmission"]["username"],
+            password=CONFIG["transmission"]["password"],
+        )
+
+        # Test connection
+        session = client.get_session()
+        print(f"  Connected to Transmission {session.version}")
+
+        # Get torrent list
+        torrents = client.get_torrents()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent = client.add_torrent(TEST_MAGNET, paused=True)
+        print(f"  Added test torrent: {torrent.name[:50]}...")
+
+        # Get status
+        status = client.get_torrent(torrent.id)
+        print(f"  Status: {status.status} ({status.percent_done * 100:.1f}%)")
+
+        # Remove it
+        client.remove_torrent(torrent.id, delete_data=True)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: Transmission is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: transmission-rpc not installed")
+        print("  Run: pip install transmission-rpc")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_deluge():
+    """Test Deluge Web UI (JSON-RPC) connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing Deluge")
+    print("=" * 50)
+
+    base_url = CONFIG["deluge"]["url"].rstrip("/")
+    password = CONFIG["deluge"]["password"]
+    rpc_url = f"{base_url}/json"
+
+    def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
+        payload = {"id": rpc_id, "method": method, "params": list(params)}
+        resp = session.post(rpc_url, json=payload, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("error"):
+            err = data["error"]
+            if isinstance(err, dict):
+                raise Exception(err.get("message") or str(err))
+            raise Exception(str(err))
+        return data.get("result")
+
+    try:
+        session = requests.Session()
+
+        # Authenticate to Deluge Web
+        if rpc_call(session, 1, "auth.login", password) is not True:
+            raise Exception("Authentication failed (check Deluge Web UI password)")
+
+        # Ensure Deluge Web is connected to a daemon
+        if rpc_call(session, 2, "web.connected") is not True:
+            hosts = rpc_call(session, 3, "web.get_hosts") or []
+            if not hosts:
+                raise Exception(
+                    "Deluge Web UI isn't connected to Deluge core (no hosts configured). "
+                    "Add/connect a daemon in Deluge Web UI → Connection Manager."
+                )
+
+            host_id = hosts[0][0]
+            for entry in hosts:
+                if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
+                    host_id = entry[0]
+                    break
+
+            rpc_call(session, 4, "web.connect", host_id)
+
+            if rpc_call(session, 5, "web.connected") is not True:
+                raise Exception(
+                    "Deluge Web UI couldn't connect to Deluge core. "
+                    "Check Deluge Web UI → Connection Manager."
+                )
+
+        version = rpc_call(session, 6, "daemon.info")
+        print(f"  Connected to Deluge {version}")
+
+        torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
+
+        if torrent_id:
+            torrent_id = str(torrent_id)
+            print(f"  Added test torrent: {torrent_id[:20]}...")
+
+            status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
+            state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
+            progress = status.get("progress", 0) if isinstance(status, dict) else 0
+            print(f"  Status: {state} ({progress:.1f}%)")
+
+            rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
+            print("  Removed test torrent")
+        else:
+            print("  WARNING: Could not add test torrent")
+
+        print("  SUCCESS: Deluge is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to Deluge Web UI")
+        print("  Is the container running? docker ps | grep deluge")
+        return False
+    except requests.exceptions.Timeout:
+        print("  ERROR: Deluge Web UI connection timed out")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "auth" in str(e).lower() or "login" in str(e).lower():
+            print("  Check Deluge Web UI password (default: deluge)")
+        return False
+
+def test_rtorrent():
+    """Test rTorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing rTorrent")
+    print("=" * 50)
+
+    try:
+        import xmlrpc.client
+
+        url = "http://localhost:8000/RPC2"
+        client = xmlrpc.client.ServerProxy(url)
+
+        # Test connection
+        version = client.system.library_version()
+        print(f"  Connected to rTorrent {version}")
+
+        # default download directory test
+        default_dir = client.directory.default()
+        print(f"  Default download directory: {default_dir}")
+
+        # Get torrent list
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+
+        label = "automated"
+
+        commands = []
+        if label:
+            commands.append(f"d.custom1.set={label}")
+
+        download_dir = "/downloads"
+        if download_dir:
+            commands.append(f"d.directory_base.set={download_dir}")
+
+        # rtorrent is weird in that it doesn't return the torrent ID/hash on add
+        client.load.start("", TEST_MAGNET, ";".join(commands))
+        
+        # but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
+        torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
+        print(f"  Added test torrent: {torrent_id}")
+
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")        
+
+        torrent_list = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal={{d.hash=,cat={torrent_id}}}"
+            "d.hash=",
+            "d.state=",
+            "d.completed_bytes=",
+            "d.size_bytes=",
+            "d.down.rate=",
+            "d.up.rate=",
+            "d.custom1=",
+            "d.complete=",
+        )
+        torrent = torrent_list[0]
+
+        if not torrent:
+            print("  ERROR: Could not find added torrent in list")
+            return False
+        
+        # let's test the base path call
+        details = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal=d.hash=,cat={torrent_id}",
+            "d.base_path=",
+        )
+
+        base_path = details[0][0] if details else None
+
+        print(f"  Base path: {base_path}")
+        client.d.erase(torrent_id)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: rTorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: xmlrpc.client not available")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Connection refused" in str(e):
+            print("  Is the container running? docker ps | grep rtorrent")
+        return False
+
+
+def main():
+    print("Download Client Test Suite")
+    print("=" * 50)
+    print("Make sure containers are running:")
+    print("  docker compose -f docker-compose.test-clients.yml up -d")
+
+    results = {}
+
+    # Test usenet clients
+    print("\n" + "=" * 50)
+    print("USENET CLIENTS")
+    print("=" * 50)
+    results["nzbget"] = test_nzbget()
+    results["sabnzbd"] = test_sabnzbd()
+
+    # Test torrent clients
+    print("\n" + "=" * 50)
+    print("TORRENT CLIENTS")
+    print("=" * 50)
+    results["qbittorrent"] = test_qbittorrent()
+    results["transmission"] = test_transmission()
+    results["deluge"] = test_deluge()
+    results["rtorrent"] = test_rtorrent()
+
+    # Summary
+    print("\n" + "=" * 50)
+    print("SUMMARY")
+    print("=" * 50)
+
+    for client, success in results.items():
+        status = "PASS" if success else "FAIL"
+        print(f"  {client}: {status}")
+
+    passed = sum(results.values())
+    total = len(results)
+    print(f"\n  Total: {passed}/{total} passed")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

shelfmark/__init__.py

@@ -0,0 +1 @@
+"""Shelfmark - book search and download service."""

shelfmark/__main__.py

@@ -0,0 +1,8 @@
+"""Package entry point for `python -m shelfmark`."""
+
+from shelfmark.main import app, socketio
+from shelfmark.config.env import FLASK_HOST, FLASK_PORT
+from shelfmark.core.config import config
+
+if __name__ == "__main__":
+    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

shelfmark/api/__init__.py

@@ -0,0 +1 @@
+"""API module - WebSocket handling."""

shelfmark/api/websocket.py

@@ -0,0 +1,251 @@
+"""WebSocket manager for real-time status updates."""
+
+import logging
+import threading
+from typing import Optional, Dict, Any, Callable, List
+
+from flask_socketio import SocketIO, join_room, leave_room
+
+logger = logging.getLogger(__name__)
+
+
+class WebSocketManager:
+    """Manages WebSocket connections and broadcasts."""
+
+    def __init__(self):
+        self.socketio: Optional[SocketIO] = None
+        self._enabled = False
+        self._connection_count = 0
+        self._connection_lock = threading.Lock()
+        self._on_first_connect_callbacks: List[Callable[[], None]] = []
+        self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
+        self._needs_rewarm = False  # Flag to trigger warmup callbacks on next connect
+        self._user_rooms: Dict[str, int] = {}  # room_name -> ref count
+        self._sid_rooms: Dict[str, str] = {}  # sid -> room_name
+        self._rooms_lock = threading.Lock()
+        self._queue_status_fn: Optional[Callable] = None  # Reference to queue_status()
+
+    def init_app(self, app, socketio: SocketIO):
+        """Initialize the WebSocket manager with Flask-SocketIO instance."""
+        self.socketio = socketio
+        self._enabled = True
+        logger.info("WebSocket manager initialized")
+
+    def register_on_first_connect(self, callback: Callable[[], None]):
+        """Register a callback for when the first client connects."""
+        self._on_first_connect_callbacks.append(callback)
+        logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
+
+    def register_on_all_disconnect(self, callback: Callable[[], None]):
+        """Register a callback for when all clients disconnect."""
+        self._on_all_disconnect_callbacks.append(callback)
+        logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
+
+    def request_warmup_on_next_connect(self):
+        """Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
+        with self._connection_lock:
+            self._needs_rewarm = True
+            logger.debug("Warmup requested for next client connect")
+
+    def client_connected(self):
+        """Track a new client connection. Call this from the connect event handler."""
+        with self._connection_lock:
+            was_zero = self._connection_count == 0
+            needs_rewarm = self._needs_rewarm
+            self._connection_count += 1
+            current_count = self._connection_count
+            # Clear rewarm flag if we're going to trigger warmup
+            if was_zero or needs_rewarm:
+                self._needs_rewarm = False
+
+        logger.debug(f"Client connected. Active connections: {current_count}")
+
+        # Trigger warmup callbacks if this is the first connection OR if rewarm was requested
+        # (rewarm is requested when bypasser shuts down due to idle while clients are connected)
+        if was_zero or needs_rewarm:
+            reason = "First client connected" if was_zero else "Rewarm requested after idle shutdown"
+            logger.info(f"{reason}, triggering warmup callbacks...")
+            for callback in self._on_first_connect_callbacks:
+                try:
+                    # Run callbacks in a separate thread to not block the connection
+                    thread = threading.Thread(target=callback, daemon=True)
+                    thread.start()
+                except Exception as e:
+                    logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}")
+
+    def client_disconnected(self):
+        """Track a client disconnection. Call this from the disconnect event handler."""
+        with self._connection_lock:
+            self._connection_count = max(0, self._connection_count - 1)
+            current_count = self._connection_count
+            is_now_zero = current_count == 0
+
+        logger.debug(f"Client disconnected. Active connections: {current_count}")
+
+        # If all clients have disconnected, trigger cleanup callbacks
+        if is_now_zero:
+            logger.info("All clients disconnected, triggering disconnect callbacks...")
+            for callback in self._on_all_disconnect_callbacks:
+                try:
+                    callback()
+                except Exception as e:
+                    logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}")
+
+    def get_connection_count(self) -> int:
+        """Get the current number of active WebSocket connections."""
+        with self._connection_lock:
+            return self._connection_count
+
+    def has_active_connections(self) -> bool:
+        """Check if there are any active WebSocket connections."""
+        return self.get_connection_count() > 0
+
+    def is_enabled(self) -> bool:
+        """Check if WebSocket is enabled and ready."""
+        return self._enabled and self.socketio is not None
+
+    def set_queue_status_fn(self, fn: Callable):
+        """Set the queue_status function reference for per-room filtering."""
+        self._queue_status_fn = fn
+
+    def _increment_user_room_locked(self, room: str):
+        self._user_rooms[room] = self._user_rooms.get(room, 0) + 1
+
+    def _decrement_user_room_locked(self, room: str):
+        count = self._user_rooms.get(room, 1) - 1
+        if count <= 0:
+            self._user_rooms.pop(room, None)
+        else:
+            self._user_rooms[room] = count
+
+    def _set_sid_room_locked(self, sid: str, room: Optional[str]):
+        current_room = self._sid_rooms.get(sid)
+        if current_room == room:
+            return
+
+        if current_room is not None:
+            leave_room(current_room, sid=sid)
+            if current_room.startswith("user_"):
+                self._decrement_user_room_locked(current_room)
+            self._sid_rooms.pop(sid, None)
+
+        if room is not None:
+            join_room(room, sid=sid)
+            self._sid_rooms[sid] = room
+            if room.startswith("user_"):
+                self._increment_user_room_locked(room)
+
+    def sync_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
+        """Ensure a SID is in exactly one room matching the current session scope."""
+        room: Optional[str] = None
+        if is_admin:
+            room = "admins"
+        elif db_user_id is not None:
+            room = f"user_{db_user_id}"
+
+        with self._rooms_lock:
+            self._set_sid_room_locked(sid, room)
+
+    def join_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
+        """Join the appropriate room based on user role."""
+        self.sync_user_room(sid, is_admin, db_user_id)
+
+    def leave_user_room(self, sid: str, is_admin: bool = False, db_user_id: Optional[int] = None):
+        """Leave whichever room the SID currently belongs to."""
+        del is_admin, db_user_id  # Backward-compatible signature; routing is SID-based.
+        with self._rooms_lock:
+            self._set_sid_room_locked(sid, None)
+
+    def broadcast_status_update(self, status_data: Dict[str, Any]):
+        """Broadcast status update to all connected clients, filtered by user room."""
+        if not self.is_enabled():
+            return
+
+        try:
+            # Admins (and no-auth users) get full status
+            self.socketio.emit('status_update', status_data, to="admins")
+
+            # Each user room gets filtered status
+            with self._rooms_lock:
+                active_rooms = list(self._user_rooms.keys())
+
+            if active_rooms and self._queue_status_fn:
+                for room in active_rooms:
+                    try:
+                        # Extract user_id from room name "user_123"
+                        uid = int(room.split("_", 1)[1])
+                        filtered = self._queue_status_fn(user_id=uid)
+                        self.socketio.emit('status_update', filtered, to=room)
+                    except Exception as e:
+                        logger.error(f"Failed to send status update for room {room}: {e}")
+
+            logger.debug("Broadcasted status update to all rooms")
+        except Exception as e:
+            logger.error(f"Error broadcasting status update: {e}")
+
+    def broadcast_download_progress(self, book_id: str, progress: float, status: str, user_id: Optional[int] = None):
+        """Broadcast download progress update for a specific book."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'book_id': book_id,
+                'progress': progress,
+                'status': status
+            }
+            # Admins always see all progress
+            self.socketio.emit('download_progress', data, to="admins")
+            # If task belongs to a specific user, send to their room too
+            if user_id is not None:
+                room = f"user_{user_id}"
+                with self._rooms_lock:
+                    if room in self._user_rooms:
+                        self.socketio.emit('download_progress', data, to=room)
+            logger.debug(f"Broadcasted progress for book {book_id}: {progress}%")
+        except Exception as e:
+            logger.error(f"Error broadcasting download progress: {e}")
+
+    def broadcast_notification(self, message: str, notification_type: str = 'info'):
+        """Broadcast a notification message to all clients."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'message': message,
+                'type': notification_type
+            }
+            # When calling socketio.emit() outside event handlers, it broadcasts by default
+            self.socketio.emit('notification', data)
+            logger.debug(f"Broadcasted notification: {message}")
+        except Exception as e:
+            logger.error(f"Error broadcasting notification: {e}")
+
+    def broadcast_search_status(
+        self,
+        source: str,
+        provider: str,
+        book_id: str,
+        message: str,
+        phase: str = 'searching'
+    ):
+        """Broadcast search status update for a release source search."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'source': source,
+                'provider': provider,
+                'book_id': book_id,
+                'message': message,
+                'phase': phase,
+            }
+            self.socketio.emit('search_status', data)
+        except Exception as e:
+            logger.error(f"Error broadcasting search status: {e}")
+
+
+# Global WebSocket manager instance
+ws_manager = WebSocketManager()

shelfmark/bypass/__init__.py

@@ -0,0 +1,5 @@
+"""Cloudflare bypass utilities."""
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""

shelfmark/bypass/external_bypasser.py

@@ -0,0 +1,128 @@
+"""External Cloudflare bypasser using FlareSolverr."""
+
+import random
+import time
+from threading import Event
+from typing import TYPE_CHECKING, Optional
+
+import requests
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.utils import normalize_http_url
+
+if TYPE_CHECKING:
+    from shelfmark.download import network
+
+logger = setup_logger(__name__)
+
+# Timeout constants (seconds)
+CONNECT_TIMEOUT = 10
+MAX_READ_TIMEOUT = 120
+READ_TIMEOUT_BUFFER = 15
+
+# Retry settings
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
+
+
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service. Returns HTML or None."""
+    raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
+    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
+    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
+
+    bypasser_url = normalize_http_url(raw_bypasser_url)
+    if not bypasser_url or not bypasser_path:
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
+        return None
+
+    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+
+    try:
+        response = requests.post(
+            f"{bypasser_url}{bypasser_path}",
+            headers={"Content-Type": "application/json"},
+            json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
+            timeout=(CONNECT_TIMEOUT, read_timeout)
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+
+        solution = result.get('solution')
+        html = solution.get('response', '') if solution else ''
+
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+
+        return html
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+
+    return None
+
+
+def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
+    """Check if operation was cancelled and raise exception if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(f"External bypasser cancelled {context}")
+        raise BypassCancelledException("Bypass cancelled")
+
+
+def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
+    """Sleep for the specified duration, checking for cancellation each second."""
+    for _ in range(int(seconds)):
+        _check_cancelled(cancel_flag, "during backoff")
+        time.sleep(1)
+    remaining = seconds - int(seconds)
+    if remaining > 0:
+        time.sleep(remaining)
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional["network.AAMirrorSelector"] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML via external bypasser with retries and mirror rotation."""
+    from shelfmark.download import network as network_module
+
+    sel = selector or network_module.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        _check_cancelled(cancel_flag, "by user")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        _sleep_with_cancellation(delay, cancel_flag)
+
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

shelfmark/bypass/fingerprint.py

@@ -0,0 +1,57 @@
+"""Browser fingerprint profile management for bypass stealth."""
+
+import random
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+COMMON_RESOLUTIONS = [
+    (1920, 1080, 0.35),  
+    (1366, 768, 0.18),   
+    (1536, 864, 0.10),   
+    (1440, 900, 0.08),   
+    (1280, 720, 0.07),   
+    (1600, 900, 0.06),   
+    (1280, 800, 0.05),   
+    (2560, 1440, 0.04),  
+    (1680, 1050, 0.04),  
+    (1920, 1200, 0.03),  
+]
+
+# Current screen size (module-level singleton)
+_current_screen_size: Optional[tuple[int, int]] = None
+
+
+def get_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    if _current_screen_size is None:
+        _current_screen_size = _generate_screen_size()
+        logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
+    return _current_screen_size
+
+
+def rotate_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    old_size = _current_screen_size
+    _current_screen_size = _generate_screen_size()
+    width, height = _current_screen_size
+
+    if old_size:
+        logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
+    else:
+        logger.info(f"Generated screen size: {width}x{height}")
+
+    return _current_screen_size
+
+
+def clear_screen_size() -> None:
+    global _current_screen_size
+    _current_screen_size = None
+
+
+def _generate_screen_size() -> tuple[int, int]:
+    resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
+    weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
+    return random.choices(resolutions, weights=weights)[0]

shelfmark/bypass/internal_bypasser.py

@@ -0,0 +1,974 @@
+import asyncio
+import os
+import random
+import signal
+import socket
+import subprocess
+import threading
+import time
+import traceback
+from datetime import datetime
+from threading import Event
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+import requests
+from seleniumbase import cdp_driver
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.bypass.fingerprint import get_screen_size
+from shelfmark.config import env
+from shelfmark.config.env import LOG_DIR
+from shelfmark.config.settings import RECORDING_DIR
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download import network
+from shelfmark.download.network import get_proxies
+
+logger = setup_logger(__name__)
+
+# Challenge detection indicators
+CLOUDFLARE_INDICATORS = [
+    "just a moment",
+    "verify you are human",
+    "verifying you are human",
+    "cloudflare.com/products/turnstile",
+]
+
+DDOS_GUARD_INDICATORS = [
+    "ddos-guard",
+    "ddos guard",
+    "checking your browser before accessing",
+    "complete the manual check to continue",
+    "could not verify your browser automatically",
+]
+
+DISPLAY = {
+    "ffmpeg": None,
+    "ffmpeg_output": None,
+}
+LOCKED = threading.Lock()
+
+
+class _CdpWorker:
+    def __init__(self) -> None:
+        self._thread: Optional[threading.Thread] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._ready = threading.Event()
+        self._lock = threading.Lock()
+
+    def _run(self) -> None:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        self._loop = loop
+        self._ready.set()
+        loop.run_forever()
+        try:
+            pending = asyncio.all_tasks(loop)
+            for task in pending:
+                task.cancel()
+            if pending:
+                loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+        except Exception:
+            pass
+        finally:
+            loop.close()
+
+    def start(self) -> None:
+        with self._lock:
+            if self._thread and self._thread.is_alive():
+                return
+            self._ready.clear()
+            self._thread = threading.Thread(
+                target=self._run,
+                name="cdp-worker",
+                daemon=True,
+            )
+            self._thread.start()
+        if not self._ready.wait(timeout=10):
+            raise RuntimeError("CDP worker loop failed to start")
+
+    def run(self, coro: Any, timeout: Optional[float] = None) -> Any:
+        self.start()
+        if not self._loop or self._loop.is_closed():
+            raise RuntimeError("CDP worker loop not available")
+        future = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        return future.result(timeout=timeout)
+
+
+_CDP_WORKER = _CdpWorker()
+
+# Cookie storage - shared with requests library for Cloudflare bypass
+# Structure: {domain: {cookie_name: {value, expiry, ...}}}
+_cf_cookies: dict[str, dict] = {}
+_cf_cookies_lock = threading.Lock()
+
+# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
+_cf_user_agents: dict[str, str] = {}
+
+# Protection cookie names we care about (Cloudflare and DDoS-Guard)
+CF_COOKIE_NAMES = {'cf_clearance', '__cf_bm', 'cf_chl_2', 'cf_chl_prog'}
+DDG_COOKIE_NAMES = {'__ddg1_', '__ddg2_', '__ddg5_', '__ddg8_', '__ddg9_', '__ddg10_', '__ddgid_', '__ddgmark_', 'ddg_last_challenge'}
+
+# Domains requiring full session cookies (not just protection cookies)
+FULL_COOKIE_DOMAINS = {'z-lib.fm', 'z-lib.gs', 'z-lib.id', 'z-library.sk', 'zlibrary-global.se'}
+
+
+def _get_base_domain(domain: str) -> str:
+    """Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
+    return '.'.join(domain.split('.')[-2:]) if '.' in domain else domain
+
+
+def _should_extract_cookie(name: str, extract_all: bool) -> bool:
+    """Determine if a cookie should be extracted based on its name."""
+    if extract_all:
+        return True
+    is_cf = name in CF_COOKIE_NAMES or name.startswith('cf_')
+    is_ddg = name in DDG_COOKIE_NAMES or name.startswith('__ddg')
+    return is_cf or is_ddg
+
+
+def _store_extracted_cookies(
+    *,
+    url: str,
+    cookies: list[Any],
+    user_agent: Optional[str] = None,
+) -> None:
+    """Store filtered bypass cookies (and optional UA) for a URL domain."""
+    parsed = urlparse(url)
+    domain = parsed.hostname or ""
+    if not domain:
+        return
+
+    base_domain = _get_base_domain(domain)
+    extract_all = base_domain in FULL_COOKIE_DOMAINS
+
+    cookies_found: dict[str, dict[str, Any]] = {}
+    for cookie in cookies:
+        name = getattr(cookie, "name", "") or ""
+        if not _should_extract_cookie(name, extract_all):
+            continue
+        expires = getattr(cookie, "expires", None)
+        if expires is not None and expires <= 0:
+            expires = None
+        cookies_found[name] = {
+            "value": getattr(cookie, "value", ""),
+            "domain": getattr(cookie, "domain", None) or domain,
+            "path": getattr(cookie, "path", None) or "/",
+            "expiry": expires,
+            "secure": bool(getattr(cookie, "secure", True)),
+            "httpOnly": True,
+        }
+
+    if not cookies_found:
+        return
+
+    with _cf_cookies_lock:
+        _cf_cookies[base_domain] = cookies_found
+        if user_agent:
+            _cf_user_agents[base_domain] = user_agent
+            logger.debug(f"Stored UA for {base_domain}: {str(user_agent)[:60]}...")
+        else:
+            logger.debug(f"No UA captured for {base_domain}")
+
+    cookie_type = "all" if extract_all else "protection"
+    logger.debug(f"Extracted {len(cookies_found)} {cookie_type} cookies for {base_domain}")
+
+
+async def _extract_cookies_from_cdp(driver, page, url: str) -> None:
+    """Extract cookies from a CDP browser after successful bypass."""
+    try:
+        try:
+            all_cookies = await driver.cookies.get_all(requests_cookie_format=True)
+        except Exception as e:
+            logger.debug(f"Failed to get cookies via CDP: {e}")
+            return
+
+        try:
+            user_agent = await page.evaluate("navigator.userAgent")
+        except Exception:
+            user_agent = None
+
+        _store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
+
+    except Exception as e:
+        logger.debug(f"Failed to extract cookies: {e}")
+
+def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
+    """Get stored cookies for a domain. Returns empty dict if none available."""
+    if not domain:
+        return {}
+
+    base_domain = _get_base_domain(domain)
+
+    with _cf_cookies_lock:
+        cookies = _cf_cookies.get(base_domain, {})
+        if not cookies:
+            return {}
+
+        cf_clearance = cookies.get('cf_clearance', {})
+        if cf_clearance:
+            expiry = cf_clearance.get('expiry')
+            if expiry is None:
+                expiry = cf_clearance.get('expires')
+            if expiry and expiry > 0 and time.time() > expiry:
+                logger.debug(f"CF cookies expired for {base_domain}")
+                _cf_cookies.pop(base_domain, None)
+                return {}
+
+        return {name: c['value'] for name, c in cookies.items()}
+
+
+def has_valid_cf_cookies(domain: str) -> bool:
+    """Check if we have valid Cloudflare cookies for a domain."""
+    return bool(get_cf_cookies_for_domain(domain))
+
+
+def get_cf_user_agent_for_domain(domain: str) -> Optional[str]:
+    """Get the User-Agent that was used during bypass for a domain."""
+    if not domain:
+        return None
+    with _cf_cookies_lock:
+        return _cf_user_agents.get(_get_base_domain(domain))
+
+
+def clear_cf_cookies(domain: str = None) -> None:
+    """Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
+    with _cf_cookies_lock:
+        if domain:
+            base_domain = _get_base_domain(domain)
+            _cf_cookies.pop(base_domain, None)
+            _cf_user_agents.pop(base_domain, None)
+        else:
+            _cf_cookies.clear()
+            _cf_user_agents.clear()
+
+
+def _cleanup_orphan_processes() -> int:
+    """Kill orphan Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode."""
+    if not env.DOCKERMODE:
+        return 0
+
+    _stop_ffmpeg_recording()
+
+    processes_to_kill = ["chrome", "chromium", "Xvfb", "ffmpeg"]
+    total_killed = 0
+
+    logger.debug("Checking for orphan processes...")
+    logger.log_resource_usage()
+
+    for proc_name in processes_to_kill:
+        try:
+            result = subprocess.run(
+                ["pgrep", "-f", proc_name],
+                capture_output=True,
+                text=True,
+                timeout=5
+            )
+            if result.returncode != 0 or not result.stdout.strip():
+                continue
+
+            pids = result.stdout.strip().split('\n')
+            count = len(pids)
+            logger.info(f"Found {count} orphan {proc_name} process(es), killing...")
+
+            kill_result = subprocess.run(
+                ["pkill", "-9", "-f", proc_name],
+                capture_output=True,
+                timeout=5
+            )
+            if kill_result.returncode == 0:
+                total_killed += count
+            else:
+                logger.warning(f"pkill for {proc_name} returned {kill_result.returncode}")
+
+        except subprocess.TimeoutExpired:
+            logger.warning(f"Timeout while checking for {proc_name} processes")
+        except Exception as e:
+            logger.debug(f"Error checking for {proc_name} processes: {e}")
+
+    if total_killed > 0:
+        time.sleep(1)
+        logger.info(f"Cleaned up {total_killed} orphan process(es)")
+        logger.log_resource_usage()
+    else:
+        logger.debug("No orphan processes found")
+
+    return total_killed
+
+async def _get_page_info(page) -> tuple[str, str, str]:
+    """Extract page title, body text, and current URL safely."""
+    try:
+        title = (await page.get_title() or "").lower()
+    except Exception:
+        title = ""
+    try:
+        body = await page.evaluate("document.body ? document.body.innerText : ''")
+        body = (body or "").lower()
+    except Exception:
+        body = ""
+    try:
+        current_url = await page.get_current_url() or ""
+    except Exception:
+        current_url = ""
+    return title, body, current_url
+
+
+def _check_indicators(title: str, body: str, indicators: list[str]) -> Optional[str]:
+    """Check if any indicator is present in title or body. Returns the found indicator or None."""
+    for indicator in indicators:
+        if indicator in title or indicator in body:
+            return indicator
+    return None
+
+def _has_cloudflare_patterns(body: str, url: str) -> bool:
+    """Check for Cloudflare-specific patterns in body or URL."""
+    return "cf-" in body or "cloudflare" in url.lower() or "/cdn-cgi/" in url
+
+async def _detect_challenge_type(page) -> str:
+    """Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'."""
+    try:
+        title, body, current_url = await _get_page_info(page)
+
+        # DDOS-Guard indicators
+        if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS):
+            logger.debug(f"DDOS-Guard indicator found: '{found}'")
+            return "ddos_guard"
+
+        # Cloudflare indicators
+        if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS):
+            logger.debug(f"Cloudflare indicator found: '{found}'")
+            return "cloudflare"
+
+        # Check URL patterns
+        if _has_cloudflare_patterns(body, current_url):
+            return "cloudflare"
+
+        return "none"
+    except Exception as e:
+        logger.warning(f"Error detecting challenge type: {e}")
+        return "none"
+
+async def _is_bypassed(page, escape_emojis: bool = True) -> bool:
+    """Check if the protection has been bypassed."""
+    try:
+        title, body, current_url = await _get_page_info(page)
+        body_len = len(body.strip())
+
+        # Long page content = probably bypassed
+        if body_len > 100000:
+            logger.debug(f"Page content too long, probably bypassed (len: {body_len})")
+            return True
+
+        # Multiple emojis = probably real content
+        if escape_emojis:
+            import emoji
+            if len(emoji.emoji_list(body)) >= 3:
+                logger.debug("Detected emojis in page, probably bypassed")
+                return True
+
+        # Check for protection indicators (means NOT bypassed)
+        if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS):
+            return False
+
+        # Cloudflare URL patterns
+        if _has_cloudflare_patterns(body, current_url):
+            logger.debug("Cloudflare patterns detected in page")
+            return False
+
+        # Page too short = still loading
+        if body_len < 50:
+            logger.debug("Page content too short, might still be loading")
+            return False
+
+        logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {body_len}")
+        return True
+
+    except Exception as e:
+        logger.warning(f"Error checking bypass status: {e}")
+        return False
+
+async def _bypass_method_humanlike(page) -> bool:
+    """Human-like behavior with scroll, wait, and reload."""
+    try:
+        logger.debug("Attempting bypass: human-like interaction")
+        await asyncio.sleep(random.uniform(6, 10))
+
+        try:
+            await page.evaluate("window.scrollTo(0, 10000);")
+            await page.wait()
+            await asyncio.sleep(random.uniform(1, 2))
+            await page.evaluate("window.scrollTo(0, 0);")
+            await page.wait()
+            await asyncio.sleep(random.uniform(2, 3))
+        except Exception as e:
+            logger.debug(f"Scroll behavior failed: {e}")
+
+        if await _is_bypassed(page):
+            return True
+
+        logger.debug("Trying page refresh...")
+        await page.reload(ignore_cache=True)
+        await asyncio.sleep(random.uniform(5, 8))
+
+        if await _is_bypassed(page):
+            return True
+
+        try:
+            await page.solve_captcha()
+            await asyncio.sleep(random.uniform(3, 5))
+        except Exception as e:
+            logger.debug(f"Final captcha click failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"Human-like method failed: {e}")
+        return False
+
+
+async def _bypass_method_cdp_solve(page) -> bool:
+    """CDP Mode with solve_captcha() - auto-detects challenge type."""
+    try:
+        logger.debug("Attempting bypass: CDP solve_captcha")
+        await page.solve_captcha()
+        await asyncio.sleep(random.uniform(3, 5))
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP solve_captcha failed: {e}")
+        return False
+
+
+CDP_CLICK_SELECTORS = [
+    "#turnstile-widget div",      # Cloudflare Turnstile
+    "#cf-turnstile div",          # Alternative CF Turnstile
+    "iframe[src*='challenges']",  # CF challenge iframe
+    "input[type='checkbox']",     # Generic checkbox (DDOS-Guard)
+    "[class*='checkbox']",        # Class-based checkbox
+    "#challenge-running",         # CF challenge indicator
+]
+
+
+async def _bypass_method_cdp_click(page) -> bool:
+    """CDP Mode with native clicking - no PyAutoGUI dependency."""
+    try:
+        logger.debug("Attempting bypass: CDP native click")
+
+        for selector in CDP_CLICK_SELECTORS:
+            try:
+                if not await page.is_element_visible(selector):
+                    continue
+
+                logger.debug(f"CDP clicking: {selector}")
+                await page.click(selector)
+                await asyncio.sleep(random.uniform(2, 4))
+
+                if await _is_bypassed(page):
+                    return True
+            except Exception as e:
+                logger.debug(f"CDP click on '{selector}' failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP Mode click failed: {e}")
+        return False
+
+
+CDP_GUI_CLICK_SELECTORS = [
+    "#turnstile-widget div",      # Cloudflare Turnstile
+    "#cf-turnstile div",          # Alternative CF Turnstile
+    "#challenge-stage div",       # CF challenge stage
+    "input[type='checkbox']",     # Generic checkbox
+    "[class*='cb-i']",            # DDOS-Guard checkbox
+]
+
+
+async def _bypass_method_cdp_gui_click(page) -> bool:
+    """CDP Mode with gui_click-style behavior."""
+    try:
+        logger.debug("Attempting bypass: CDP gui_click (mouse-based)")
+
+        try:
+            logger.debug("Trying solve_captcha()")
+            await page.solve_captcha()
+            await asyncio.sleep(random.uniform(3, 5))
+
+            if await _is_bypassed(page):
+                return True
+        except Exception as e:
+            logger.debug(f"solve_captcha() failed: {e}")
+
+        for selector in CDP_GUI_CLICK_SELECTORS:
+            try:
+                if not await page.is_element_visible(selector):
+                    continue
+
+                logger.debug(f"CDP click_with_offset: {selector}")
+                await page.click_with_offset(selector, 0, 0, center=True)
+                await asyncio.sleep(random.uniform(3, 5))
+
+                if await _is_bypassed(page):
+                    return True
+            except Exception as e:
+                logger.debug(f"CDP gui_click on '{selector}' failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP Mode gui_click failed: {e}")
+        return False
+
+
+BYPASS_METHODS = [
+    _bypass_method_cdp_solve,
+    _bypass_method_cdp_gui_click,
+    _bypass_method_cdp_click,
+    _bypass_method_humanlike,
+]
+
+MAX_CONSECUTIVE_SAME_CHALLENGE = 3
+
+
+def _check_cancellation(cancel_flag: Optional[Event], message: str) -> None:
+    """Check if cancellation was requested and raise if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(message)
+        raise BypassCancelledException("Bypass cancelled")
+
+
+async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional[Event] = None) -> bool:
+    """Attempt to bypass Cloudflare/DDOS-Guard protection using multiple methods."""
+    max_retries = max_retries if max_retries is not None else app_config.MAX_RETRY
+
+    last_challenge_type = None
+    consecutive_same_challenge = 0
+    # Allow at least one full pass through all bypass methods before aborting due to a "stuck" challenge.
+    min_same_challenge_before_abort = max(MAX_CONSECUTIVE_SAME_CHALLENGE, len(BYPASS_METHODS) + 1)
+
+    for try_count in range(max_retries):
+        _check_cancellation(cancel_flag, "Bypass cancelled by user")
+
+        if await _is_bypassed(page):
+            if try_count == 0:
+                logger.info("Page already bypassed")
+            return True
+
+        challenge_type = await _detect_challenge_type(page)
+        logger.debug(f"Challenge detected: {challenge_type}")
+
+        # No challenge detected but page doesn't look bypassed - wait and retry
+        if challenge_type == "none":
+            logger.info("No challenge detected, waiting for page to settle...")
+            await asyncio.sleep(random.uniform(2, 3))
+            if await _is_bypassed(page):
+                return True
+            # Try a simple refresh instead of captcha methods
+            try:
+                await page.reload(ignore_cache=True)
+                await asyncio.sleep(random.uniform(1, 2))
+                if await _is_bypassed(page):
+                    logger.info("Bypass successful after refresh")
+                    return True
+            except Exception as e:
+                logger.debug(f"Refresh during no-challenge wait failed: {e}")
+            continue
+
+        if challenge_type == last_challenge_type:
+            consecutive_same_challenge += 1
+            if consecutive_same_challenge >= min_same_challenge_before_abort:
+                logger.warning(
+                    f"Same challenge ({challenge_type}) detected {consecutive_same_challenge} times - aborting"
+                )
+                return False
+        else:
+            consecutive_same_challenge = 1
+        last_challenge_type = challenge_type
+
+        method = BYPASS_METHODS[try_count % len(BYPASS_METHODS)]
+        logger.info(f"Bypass attempt {try_count + 1}/{max_retries} using {method.__name__}")
+
+        if try_count > 0:
+            wait_time = min(random.uniform(2, 4) * try_count, 12)
+            logger.info(f"Waiting {wait_time:.1f}s before trying...")
+            for _ in range(int(wait_time)):
+                _check_cancellation(cancel_flag, "Bypass cancelled during wait")
+                await asyncio.sleep(1)
+            await asyncio.sleep(wait_time - int(wait_time))
+
+        try:
+            if await method(page):
+                logger.info(f"Bypass successful using {method.__name__}")
+                return True
+        except BypassCancelledException:
+            raise
+        except Exception as e:
+            logger.warning(f"Exception in {method.__name__}: {e}")
+
+        logger.info(f"Bypass method {method.__name__} failed.")
+
+    logger.warning("Exceeded maximum retries. Bypass failed.")
+    return False
+
+def _get_browser_args() -> list[str]:
+    """Build extra Chrome arguments, pre-resolving hostnames via patched DNS.
+
+    Pre-resolves AA hostnames and passes IPs to Chrome via --host-resolver-rules,
+    bypassing Chrome's DNS entirely for those hosts.
+    """
+    arguments = [
+        "--ignore-certificate-errors",
+        "--ignore-ssl-errors",
+        "--allow-running-insecure-content",
+        "--ignore-certificate-errors-spki-list",
+        "--ignore-certificate-errors-skip-list",
+        # Chrome 144+ disabled automatic SwiftShader fallback for WebGL (security reasons).
+        # Without this flag, WebGL is broken in headless/Docker which triggers bot detection.
+        # See: https://issues.chromium.org/issues/40277080
+        "--enable-unsafe-swiftshader",
+    ]
+
+    if app_config.get("DEBUG", False):
+        arguments.extend([
+            "--enable-logging",
+            "--v=1",
+            "--log-file=" + str(LOG_DIR / "chrome_browser.log")
+        ])
+
+    host_rules = _build_host_resolver_rules()
+    if host_rules:
+        arguments.append(f'--host-resolver-rules={", ".join(host_rules)}')
+        logger.debug(f"Chrome: Using host resolver rules for {len(host_rules)} hosts")
+    else:
+        logger.warning("Chrome: No hosts could be pre-resolved")
+
+    return arguments
+
+
+def _build_host_resolver_rules() -> list[str]:
+    """Pre-resolve AA hostnames and build Chrome host resolver rules."""
+    host_rules = []
+
+    try:
+        for url in network.get_available_aa_urls():
+            hostname = urlparse(url).hostname
+            if not hostname:
+                continue
+
+            try:
+                results = socket.getaddrinfo(hostname, 443, socket.AF_INET)
+                if results:
+                    ip = results[0][4][0]
+                    host_rules.append(f"MAP {hostname} {ip}")
+                    logger.debug(f"Chrome: Pre-resolved {hostname} -> {ip}")
+                else:
+                    logger.warning(f"Chrome: No addresses returned for {hostname}")
+            except socket.gaierror as e:
+                logger.warning(f"Chrome: Could not pre-resolve {hostname}: {e}")
+    except Exception as e:
+        logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}")
+
+    return host_rules
+
+DRIVER_RESET_ERRORS = {"ProtocolException", "RuntimeError", "TimeoutError"}
+
+
+async def _get(url: str, driver, cancel_flag: Optional[Event] = None) -> str:
+    """Fetch URL with Cloudflare bypass using a CDP browser."""
+    _check_cancellation(cancel_flag, "Bypass cancelled before starting")
+
+    logger.debug(f"CDP_GET: {url}")
+
+    logger.debug("Opening URL with SeleniumBase CDP...")
+    page = await driver.get(url)
+    try:
+        await page.wait()
+    except Exception:
+        pass
+
+    _check_cancellation(cancel_flag, "Bypass cancelled after page load")
+
+    try:
+        current_url = await page.get_current_url()
+        title = await page.get_title()
+        logger.debug(f"Page loaded - URL: {current_url}, Title: {title}")
+    except Exception as e:
+        logger.debug(f"Could not get page info: {e}")
+
+    logger.debug("Starting bypass process...")
+    if await _bypass(page, cancel_flag=cancel_flag):
+        await _extract_cookies_from_cdp(driver, page, url)
+        return await page.get_page_source()
+
+    logger.warning("Bypass completed but page still shows protection")
+    try:
+        body = await page.evaluate("document.body ? document.body.innerText : ''")
+        if body:
+            logger.debug(f"Page content: {body[:500]}..." if len(body) > 500 else body)
+    except Exception:
+        pass
+
+    return ""
+
+
+def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = None) -> str:
+    """Fetch a URL with protection bypass. Creates fresh Chrome instance for each bypass."""
+    retry = retry if retry is not None else app_config.MAX_RETRY
+
+    with LOCKED:
+        # Try cookies first - another request may have completed bypass while waiting
+        cached_result = _try_with_cached_cookies(url, urlparse(url).hostname or "")
+        if cached_result:
+            return cached_result
+
+        async def _run_bypass() -> str:
+            driver = None
+            try:
+                driver = await _create_cdp_browser(url)
+
+                for attempt in range(retry):
+                    _check_cancellation(cancel_flag, "Bypass cancelled before attempt")
+
+                    try:
+                        result = await _get(url, driver, cancel_flag)
+                        if result:
+                            return result
+                    except BypassCancelledException:
+                        raise
+                    except Exception as e:
+                        error_details = f"{type(e).__name__}: {e}"
+                        logger.warning(f"Bypass failed (attempt {attempt + 1}/{retry}): {error_details}")
+                        logger.debug(f"Stack trace: {traceback.format_exc()}")
+
+                        # On CDP errors, quit and create a fresh browser
+                        if type(e).__name__ in DRIVER_RESET_ERRORS:
+                            logger.info("Restarting Chrome due to browser error...")
+                            await _close_cdp_driver(driver)
+                            driver = await _create_cdp_browser(url)
+
+                logger.error(f"Bypass failed after {retry} attempts")
+                return ""
+            finally:
+                if driver:
+                    await _close_cdp_driver(driver)
+
+        return _CDP_WORKER.run(_run_bypass())
+
+def _get_proxy_string(url: str) -> Optional[str]:
+    """Return a single proxy string for CDP, honoring NO_PROXY."""
+    proxies = get_proxies(url)
+    if not proxies:
+        return None
+    proxy_url = proxies.get("https") or proxies.get("http")
+    return proxy_url or None
+
+
+async def _create_cdp_browser(url: str) -> Any:
+    """Create a fresh CDP browser instance."""
+    browser_args = _get_browser_args()
+    screen_width, screen_height = get_screen_size()
+    display_width = screen_width + 100
+    display_height = screen_height + 150
+    proxy = _get_proxy_string(url)
+
+    logger.debug(f"Creating Pure CDP browser with args: {browser_args}")
+    logger.debug(f"Browser screen size: {screen_width}x{screen_height}")
+
+    driver = await cdp_driver.start_async(
+        headless=False,
+        headed=False,
+        xvfb=True,
+        xvfb_metrics=f"{display_width},{display_height}",
+        sandbox=False,
+        lang="en",
+        incognito=True,
+        ad_block=True,
+        proxy=proxy,
+        browser_args=browser_args,
+    )
+
+    try:
+        await driver.page.set_window_rect(0, 0, screen_width, screen_height)
+    except Exception as e:
+        logger.debug(f"Failed to set window size: {e}")
+
+    # Start FFmpeg recording if debug mode (record each bypass session)
+    if app_config.get("DEBUG", False) and not DISPLAY.get("ffmpeg"):
+        _start_ffmpeg_recording(display=os.environ.get("DISPLAY", ":0"))
+
+    await asyncio.sleep(app_config.DEFAULT_SLEEP)
+    logger.info("Chrome browser ready (Pure CDP)")
+    logger.log_resource_usage()
+    return driver
+
+
+async def _close_cdp_driver(driver) -> None:
+    """Close CDP connections and stop the browser."""
+    if not driver:
+        return
+
+    logger.debug("Quitting Chrome browser (CDP)...")
+
+    _stop_ffmpeg_recording()
+
+    try:
+        connections = []
+        if hasattr(driver, "connection") and driver.connection:
+            connections.append(driver.connection)
+        if hasattr(driver, "targets") and driver.targets:
+            connections.extend(driver.targets)
+        for conn in connections:
+            try:
+                await conn.aclose()
+            except Exception as e:
+                logger.debug(f"Failed to close websocket connection: {e}")
+    except Exception as e:
+        logger.debug(f"Error during connection cleanup: {e}")
+
+    try:
+        driver.stop()
+        logger.debug("Stopped CDP browser")
+    except Exception as e:
+        logger.debug(f"CDP stop: {e}")
+
+    if env.DOCKERMODE:
+        await asyncio.sleep(0.3)
+        try:
+            pid = getattr(driver, "_process_pid", None)
+
+            def _pid_alive(check_pid: int) -> bool:
+                try:
+                    os.kill(check_pid, 0)
+                except ProcessLookupError:
+                    return False
+                except PermissionError:
+                    return True
+                return True
+
+            if pid and _pid_alive(pid):
+                try:
+                    os.kill(pid, signal.SIGTERM)
+                    await asyncio.sleep(0.1)
+                    if _pid_alive(pid):
+                        os.kill(pid, signal.SIGKILL)
+                    logger.debug(f"Killed Chrome pid {pid}")
+                except Exception as e:
+                    logger.debug(f"Failed to kill Chrome pid {pid}: {e}")
+        except Exception as e:
+            logger.debug(f"Process cleanup failed: {e}")
+
+    logger.log_resource_usage()
+
+
+def _start_ffmpeg_recording(display: str) -> None:
+    """Start FFmpeg screen recording for debug mode."""
+    global DISPLAY
+    RECORDING_DIR.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
+    output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
+
+    screen_width, screen_height = get_screen_size()
+    display_width = screen_width + 100
+    display_height = screen_height + 150
+
+    ffmpeg_cmd = [
+        "ffmpeg", "-y", "-f", "x11grab",
+        "-video_size", f"{display_width}x{display_height}",
+        "-i", display,
+        "-c:v", "libx264", "-preset", "ultrafast",
+        "-maxrate", "700k", "-bufsize", "1400k", "-crf", "36",
+        "-pix_fmt", "yuv420p", "-tune", "animation",
+        "-x264-params", "bframes=0:deblock=-1,-1",
+        "-r", "15", "-an",
+        output_file.as_posix(),
+        "-nostats", "-loglevel", "0"
+    ]
+    logger.debug("Starting FFmpeg recording to %s", output_file)
+    logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
+    DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
+    DISPLAY["ffmpeg_output"] = output_file
+
+
+def _stop_ffmpeg_recording() -> None:
+    """Stop FFmpeg screen recording if running."""
+    import signal
+    global DISPLAY
+    proc = DISPLAY.get("ffmpeg")
+    output_file = DISPLAY.get("ffmpeg_output")
+    if not proc:
+        return
+    if proc.poll() is not None:
+        logger.debug("FFmpeg already stopped")
+        DISPLAY["ffmpeg"] = None
+        DISPLAY["ffmpeg_output"] = None
+        return
+    try:
+        proc.send_signal(signal.SIGINT)
+        proc.wait(timeout=5)
+        logger.debug("Stopped ffmpeg recording")
+    except Exception as e:
+        logger.debug(f"ffmpeg stop: {e}")
+        try:
+            proc.terminate()
+            proc.wait(timeout=2)
+        except Exception:
+            pass
+        try:
+            proc.kill()
+        except Exception:
+            pass
+    DISPLAY["ffmpeg"] = None
+    DISPLAY["ffmpeg_output"] = None
+
+
+def _try_with_cached_cookies(url: str, hostname: str) -> Optional[str]:
+    """Attempt request with cached cookies before using Chrome."""
+    cookies = get_cf_cookies_for_domain(hostname)
+    if not cookies:
+        return None
+
+    try:
+        headers = {}
+        stored_ua = get_cf_user_agent_for_domain(hostname)
+        if stored_ua:
+            headers['User-Agent'] = stored_ua
+
+        logger.debug(f"Trying request with cached cookies: {url}")
+        response = requests.get(url, cookies=cookies, headers=headers, proxies=get_proxies(url), timeout=(5, 10))
+        if response.status_code == 200:
+            logger.debug("Cached cookies worked, skipped Chrome bypass")
+            return response.text
+    except Exception:
+        pass
+
+    return None
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional[network.AAMirrorSelector] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML content from a URL using the internal Cloudflare Bypasser."""
+    sel = selector or network.AAMirrorSelector()
+    attempt_url = sel.rewrite(url)
+    hostname = urlparse(attempt_url).hostname or ""
+
+    cached_result = _try_with_cached_cookies(attempt_url, hostname)
+    if cached_result:
+        return cached_result
+
+    try:
+        response_html = get(attempt_url, cancel_flag=cancel_flag)
+    except BypassCancelledException:
+        raise
+    except Exception:
+        _check_cancellation(cancel_flag, "Bypass cancelled")
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            attempt_url = sel.rewrite(url)
+            response_html = get(attempt_url, cancel_flag=cancel_flag)
+        else:
+            raise
+
+    if not response_html.strip():
+        raise requests.exceptions.RequestException("Failed to bypass Cloudflare")
+
+    return response_html

shelfmark/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration module - environment variables and settings."""

shelfmark/config/booklore_settings.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.outputs.booklore import (
+    BookloreConfig,
+    BookloreError,
+    booklore_list_libraries,
+    booklore_login,
+)
+
+logger = setup_logger(__name__)
+
+_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
+    "key": None,
+    "library_options": [],
+    "path_options": [],
+}
+
+
+def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
+    return f"{base_url}|{username}|{hash(password)}"
+
+
+def _get_booklore_select_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    # library_id/path_id are not used for login/library listing
+    booklore_config = BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=1,
+        path_id=1,
+        verify_tls=True,
+        refresh_after_upload=True,
+    )
+
+    token = booklore_login(booklore_config)
+    libraries = booklore_list_libraries(booklore_config, token) or []
+    logger.debug("Booklore libraries response: %s", libraries)
+
+    library_options: list[dict[str, Any]] = []
+    path_options: list[dict[str, Any]] = []
+
+    for library in libraries:
+        if not isinstance(library, dict):
+            continue
+
+        library_id = library.get("id")
+        if library_id is None:
+            continue
+
+        library_name = str(library.get("name") or f"Library {library_id}")
+        library_id_str = str(library_id)
+
+        library_options.append({"value": library_id_str, "label": library_name})
+
+        paths = library.get("paths") or []
+        if not isinstance(paths, list):
+            continue
+
+        for path in paths:
+            if not isinstance(path, dict):
+                continue
+
+            path_id = path.get("id")
+            if path_id is None:
+                continue
+
+            path_label = str(path.get("path") or f"Path {path_id}")
+            path_options.append(
+                {
+                    "value": str(path_id),
+                    "label": f"{library_name}: {path_label}",
+                    "childOf": library_id_str,
+                }
+            )
+
+    logger.debug(
+        "Booklore options built: libraries=%d paths=%d",
+        len(library_options),
+        len(path_options),
+    )
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    _BOOKLORE_OPTIONS_CACHE.update(
+        {
+            "key": cache_key,
+            "library_options": library_options,
+            "path_options": path_options,
+        }
+    )
+
+    return library_options, path_options
+
+
+def _get_booklore_cached_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+        return (
+            _BOOKLORE_OPTIONS_CACHE.get("library_options", []),
+            _BOOKLORE_OPTIONS_CACHE.get("path_options", []),
+        )
+
+    return _get_booklore_select_options(base_url, username, password)
+
+
+def get_booklore_library_options() -> list[dict[str, Any]]:
+    """Build Booklore library options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        library_options, _ = _get_booklore_cached_options(base_url, username, password)
+        return library_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore libraries: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
+        return []
+
+
+def get_booklore_path_options() -> list[dict[str, Any]]:
+    """Build Booklore path options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        _, path_options = _get_booklore_cached_options(base_url, username, password)
+        return path_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore paths: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
+        return []
+
+
+def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test the Booklore connection using current form values."""
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
+    password = _get_value("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        return {"success": False, "message": "Booklore URL is required"}
+    if not username:
+        return {"success": False, "message": "Booklore username is required"}
+    if not password:
+        return {"success": False, "message": "Booklore password is required"}
+
+    try:
+        library_options, _ = _get_booklore_select_options(base_url, username, password)
+
+        message = "Connected to Booklore"
+        if library_options:
+            message = f"Connected to Booklore ({len(library_options)} libraries)"
+
+        return {"success": True, "message": message}
+    except BookloreError as exc:
+        return {"success": False, "message": str(exc)}

shelfmark/config/email_settings.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.download.outputs.email import EmailOutputError, build_email_smtp_config, test_smtp_connection
+
+
+def test_email_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test SMTP connectivity using current form values (including unsaved changes)."""
+
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    settings = {
+        "EMAIL_SMTP_HOST": _get_value("EMAIL_SMTP_HOST", ""),
+        "EMAIL_SMTP_PORT": _get_value("EMAIL_SMTP_PORT", 587),
+        "EMAIL_SMTP_SECURITY": _get_value("EMAIL_SMTP_SECURITY", "starttls"),
+        "EMAIL_SMTP_USERNAME": _get_value("EMAIL_SMTP_USERNAME", ""),
+        "EMAIL_SMTP_PASSWORD": _get_value("EMAIL_SMTP_PASSWORD", ""),
+        "EMAIL_FROM": _get_value("EMAIL_FROM", ""),
+        "EMAIL_SUBJECT_TEMPLATE": _get_value("EMAIL_SUBJECT_TEMPLATE", "{Title}"),
+        "EMAIL_SMTP_TIMEOUT_SECONDS": _get_value("EMAIL_SMTP_TIMEOUT_SECONDS", 60),
+        "EMAIL_ALLOW_UNVERIFIED_TLS": _get_value("EMAIL_ALLOW_UNVERIFIED_TLS", False),
+    }
+
+    try:
+        smtp_config = build_email_smtp_config(settings)
+        test_smtp_connection(smtp_config)
+        return {"success": True, "message": "Connected to SMTP server"}
+    except EmailOutputError as exc:
+        return {"success": False, "message": str(exc)}
+    except Exception as exc:
+        return {"success": False, "message": f"SMTP test failed: {exc}"}
+

shelfmark/config/env.py

@@ -0,0 +1,162 @@
+"""Bootstrap environment variables. No local dependencies - import first."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+
+def string_to_bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ["true", "yes", "1", "y"]
+
+
+def _read_debug_from_config() -> bool:
+    """Read DEBUG from env var or config file (import-time safe)."""
+    env_debug = os.environ.get("DEBUG")
+    if env_debug is not None:
+        return string_to_bool(env_debug)
+
+    # Try to read from config file
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    config_file = config_dir / "plugins" / "advanced.json"
+
+    if config_file.exists():
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+                if "DEBUG" in config:
+                    return bool(config["DEBUG"])
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return False
+
+
+def _is_sqlite_file(path: Path) -> bool:
+    """Check if a file is a valid SQLite database by reading magic bytes."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(16)
+            return header[:16] == b"SQLite format 3\x00"
+    except (OSError, PermissionError):
+        return False
+
+
+def _resolve_cwa_db_path() -> Path | None:
+    """Resolve CWA database path from env var or default location."""
+    env_path = os.getenv("CWA_DB_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file() and _is_sqlite_file(path):
+            return path
+
+    # Check default mount path
+    default_path = Path("/auth/app.db")
+    if default_path.exists() and default_path.is_file() and _is_sqlite_file(default_path):
+        return default_path
+
+    return None
+
+
+def _is_config_dir_writable() -> bool:
+    """Check if the config directory exists and is writable."""
+    try:
+        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
+            return False
+        test_file = CONFIG_DIR / ".write_test"
+        test_file.touch()
+        test_file.unlink()
+        return True
+    except (OSError, PermissionError):
+        return False
+
+
+def is_covers_cache_enabled() -> bool:
+    """Check if cover caching is enabled (requires setting + writable config dir)."""
+    from shelfmark.core.config import config
+    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
+    return setting_enabled and _is_config_dir_writable()
+
+
+# =============================================================================
+# Bootstrap paths - needed before settings registry is available
+# =============================================================================
+
+CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
+LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
+LOG_DIR = LOG_ROOT / "shelfmark"
+LOG_FILE = LOG_DIR / "shelfmark.log"
+TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
+INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
+
+
+# =============================================================================
+# Logger configuration - needed before settings registry is available
+# =============================================================================
+
+DEBUG = _read_debug_from_config()
+LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
+ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+
+
+# =============================================================================
+# Flask configuration - needed before app starts
+# =============================================================================
+
+FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
+
+
+# =============================================================================
+# Authentication
+# =============================================================================
+
+SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
+SESSION_COOKIE_NAME = "shelfmark_session"
+CWA_DB_PATH = _resolve_cwa_db_path()
+
+
+# =============================================================================
+# Version information from Docker build
+# =============================================================================
+
+BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
+RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
+
+
+# =============================================================================
+# Capability detection - runtime checks, not user-configurable
+# =============================================================================
+
+DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
+TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
+USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
+
+
+# =============================================================================
+# Onboarding
+# =============================================================================
+
+# Set to false to skip the onboarding wizard entirely (useful for ephemeral storage)
+ONBOARDING = string_to_bool(os.getenv("ONBOARDING", "true"))
+
+
+# =============================================================================
+# Debug/development settings
+# =============================================================================
+
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
+
+
+# =============================================================================
+# Legacy migration support - will be removed in future version
+# =============================================================================
+
+# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
+# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
+_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))

shelfmark/config/migrations.py

@@ -0,0 +1,123 @@
+"""Configuration migration helpers."""
+
+import json
+from typing import Any, Callable
+
+
+_DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
+    "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
+    "CWA_RESTRICT_SETTINGS_TO_ADMIN",
+    "RESTRICT_SETTINGS_TO_ADMIN",
+)
+
+
+def _as_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+    return bool(value)
+
+
+def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None:
+    """Pick the best legacy admin-restriction value to migrate."""
+    auth_method = str(config.get("AUTH_METHOD", "")).strip().lower()
+
+    if (
+        auth_method == "proxy"
+        and "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config
+    ):
+        return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if auth_method == "cwa" and "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    return None
+
+
+def migrate_security_settings(
+    *,
+    load_security_config: Callable[[], dict[str, Any]],
+    load_users_config: Callable[[], dict[str, Any]],
+    save_users_config: Callable[[dict[str, Any]], None],
+    ensure_config_dir: Callable[[], None],
+    get_config_path: Callable[[], Any],
+    sync_builtin_admin_user: Callable[[str, str], None],
+    logger: Any,
+) -> None:
+    """Migrate legacy security keys and sync builtin admin credentials."""
+    try:
+        config = load_security_config()
+        users_config = load_users_config()
+        migrated_security = False
+        migrated_users = False
+
+        if "USE_CWA_AUTH" in config:
+            old_value = config.pop("USE_CWA_AUTH")
+            if "AUTH_METHOD" not in config:
+                if old_value:
+                    config["AUTH_METHOD"] = "cwa"
+                    logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
+                else:
+                    if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                        config["AUTH_METHOD"] = "builtin"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
+                    else:
+                        config["AUTH_METHOD"] = "none"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
+                migrated_security = True
+            else:
+                logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
+                migrated_security = True
+
+        if "RESTRICT_SETTINGS_TO_ADMIN" not in users_config:
+            legacy_restrict = _pick_legacy_settings_restriction(config)
+            if legacy_restrict is not None:
+                save_users_config({"RESTRICT_SETTINGS_TO_ADMIN": legacy_restrict})
+                migrated_users = True
+                logger.info(
+                    "Migrated legacy settings-admin restriction to users.RESTRICT_SETTINGS_TO_ADMIN="
+                    f"{legacy_restrict}"
+                )
+
+        for deprecated_key in _DEPRECATED_SETTINGS_RESTRICTION_KEYS:
+            if deprecated_key in config:
+                config.pop(deprecated_key, None)
+                migrated_security = True
+                logger.info(f"Removed deprecated security setting: {deprecated_key}")
+
+        try:
+            sync_builtin_admin_user(
+                config.get("BUILTIN_USERNAME", ""),
+                config.get("BUILTIN_PASSWORD_HASH", ""),
+            )
+        except Exception as exc:
+            logger.error(
+                "Failed to sync builtin credentials to users database during migration: "
+                f"{exc}"
+            )
+
+        if migrated_security:
+            ensure_config_dir()
+            config_path = get_config_path()
+            with open(config_path, "w") as f:
+                json.dump(config, f, indent=2)
+            logger.info("Security settings migration completed successfully")
+        elif migrated_users:
+            logger.info("Users settings migration completed successfully")
+        else:
+            logger.debug("No security settings migration needed")
+
+    except FileNotFoundError:
+        logger.debug("No existing security config file found - nothing to migrate")
+    except Exception as exc:
+        logger.error(f"Failed to migrate security settings: {exc}")

shelfmark/config/notifications_settings.py

@@ -0,0 +1,337 @@
+"""Notifications settings tab registration."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlsplit
+
+from shelfmark.core.notifications import NotificationEvent, send_test_notification
+from shelfmark.core.settings_registry import (
+    ActionButton,
+    HeadingField,
+    TableField,
+    load_config_file,
+    register_on_save,
+    register_settings,
+)
+
+_URL_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*$")
+
+_ROUTE_EVENT_ALL = "all"
+_ADMIN_EVENT_OPTIONS = [
+    {"value": NotificationEvent.REQUEST_CREATED.value, "label": "New request submitted"},
+    {"value": NotificationEvent.REQUEST_FULFILLED.value, "label": "Request approved"},
+    {"value": NotificationEvent.REQUEST_REJECTED.value, "label": "Request rejected"},
+    {"value": NotificationEvent.DOWNLOAD_COMPLETE.value, "label": "Download complete"},
+    {"value": NotificationEvent.DOWNLOAD_FAILED.value, "label": "Download failed"},
+]
+_ROUTE_EVENT_OPTIONS = [
+    {"value": _ROUTE_EVENT_ALL, "label": "All"},
+    *_ADMIN_EVENT_OPTIONS,
+]
+_ROUTE_EVENT_ORDER = [option["value"] for option in _ROUTE_EVENT_OPTIONS]
+_ROUTE_EVENT_INDEX = {event: index for index, event in enumerate(_ROUTE_EVENT_ORDER)}
+_ALLOWED_ROUTE_EVENTS = set(_ROUTE_EVENT_ORDER)
+
+_DEFAULT_ROUTE_ROWS = [{"event": [_ROUTE_EVENT_ALL], "url": ""}]
+
+
+def _looks_like_apprise_url(url: str) -> bool:
+    split = urlsplit(url)
+    if not split.scheme:
+        return False
+    if not _URL_SCHEME_RE.match(split.scheme):
+        return False
+    return " " not in url
+
+
+def _coerce_route_rows(value: Any) -> list[dict[str, Any]]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [row for row in value if isinstance(row, dict)]
+    if isinstance(value, dict):
+        return [value]
+    return []
+
+
+def _coerce_route_event_values(value: Any) -> list[Any]:
+    if isinstance(value, list):
+        return value
+    if isinstance(value, (tuple, set)):
+        return list(value)
+    return [value]
+
+
+def _normalize_route_events(value: Any) -> list[str]:
+    normalized: list[str] = []
+    seen: set[str] = set()
+
+    for raw_event in _coerce_route_event_values(value):
+        event = str(raw_event or "").strip().lower()
+        if not event or event not in _ALLOWED_ROUTE_EVENTS:
+            continue
+        if event in seen:
+            continue
+        seen.add(event)
+        normalized.append(event)
+
+    if _ROUTE_EVENT_ALL in seen:
+        return [_ROUTE_EVENT_ALL]
+
+    return sorted(normalized, key=lambda event: _ROUTE_EVENT_INDEX[event])
+
+
+def _normalize_routes(value: Any) -> list[dict[str, Any]]:
+    normalized: list[dict[str, Any]] = []
+    seen: set[tuple[tuple[str, ...], str]] = set()
+
+    for row in _coerce_route_rows(value):
+        events = _normalize_route_events(row.get("event"))
+        if not events:
+            continue
+
+        url = str(row.get("url") or "").strip()
+        key = (tuple(events), url)
+        if key in seen:
+            continue
+        seen.add(key)
+
+        normalized.append({"event": events, "url": url})
+
+    return normalized
+
+
+def _count_invalid_route_events(value: Any) -> int:
+    invalid = 0
+    for row in _coerce_route_rows(value):
+        raw_events = _coerce_route_event_values(row.get("event"))
+        if not raw_events:
+            invalid += 1
+            continue
+
+        for raw_event in raw_events:
+            event = str(raw_event or "").strip().lower()
+            if not event or event not in _ALLOWED_ROUTE_EVENTS:
+                invalid += 1
+    return invalid
+
+
+def _count_invalid_route_urls(routes: list[dict[str, Any]]) -> int:
+    return sum(1 for row in routes if row["url"] and not _looks_like_apprise_url(row["url"]))
+
+
+def _ensure_default_route_row(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return routes if routes else [dict(row) for row in _DEFAULT_ROUTE_ROWS]
+
+
+def _extract_unique_route_urls(routes: list[dict[str, Any]]) -> list[str]:
+    urls: list[str] = []
+    seen: set[str] = set()
+    for row in routes:
+        url = row.get("url", "")
+        if not url:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        urls.append(url)
+    return urls
+
+
+def build_notification_test_result(routes_input: Any, *, scope_label: str) -> dict[str, Any]:
+    invalid_event_count = _count_invalid_route_events(routes_input)
+    if invalid_event_count:
+        return {
+            "success": False,
+            "message": (
+                f"Found {invalid_event_count} invalid {scope_label} notification route event value(s). "
+                "Fix route events before running a test."
+            ),
+        }
+
+    normalized_routes = _normalize_routes(routes_input)
+    invalid_url_count = _count_invalid_route_urls(normalized_routes)
+    if invalid_url_count:
+        return {
+            "success": False,
+            "message": (
+                f"Found {invalid_url_count} invalid {scope_label} notification URL(s). "
+                "Fix route URLs before running a test."
+            ),
+        }
+
+    urls = _extract_unique_route_urls(normalized_routes)
+    if not urls:
+        return {
+            "success": False,
+            "message": f"Add at least one {scope_label} notification URL route first.",
+        }
+
+    return send_test_notification(urls)
+
+
+def normalize_notification_routes(value: Any) -> list[dict[str, Any]]:
+    """Normalize route table rows for notification preferences."""
+    return _normalize_routes(value)
+
+
+def is_valid_notification_url(url: str) -> bool:
+    """Shared URL validation for notifications preferences."""
+    return _looks_like_apprise_url(url)
+
+
+def _on_save_notifications(values: dict[str, Any]) -> dict[str, Any]:
+    existing = load_config_file("notifications")
+    effective: dict[str, Any] = dict(existing)
+    effective.update(values)
+
+    admin_routes_input = effective.get("ADMIN_NOTIFICATION_ROUTES", [])
+    invalid_admin_event_count = _count_invalid_route_events(admin_routes_input)
+    if invalid_admin_event_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_admin_event_count} invalid global notification route event value(s)."
+            ),
+            "values": values,
+        }
+
+    normalized_admin_routes = _normalize_routes(admin_routes_input)
+    invalid_admin_url_count = _count_invalid_route_urls(normalized_admin_routes)
+    if invalid_admin_url_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_admin_url_count} invalid global notification URL(s). "
+                "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+            ),
+            "values": values,
+        }
+
+    user_routes_input = effective.get("USER_NOTIFICATION_ROUTES", [])
+    invalid_user_event_count = _count_invalid_route_events(user_routes_input)
+    if invalid_user_event_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_user_event_count} invalid personal notification route event value(s)."
+            ),
+            "values": values,
+        }
+
+    normalized_user_routes = _normalize_routes(user_routes_input)
+    invalid_user_url_count = _count_invalid_route_urls(normalized_user_routes)
+    if invalid_user_url_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_user_url_count} invalid personal notification URL(s). "
+                "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+            ),
+            "values": values,
+        }
+
+    admin_routes_touched = "ADMIN_NOTIFICATION_ROUTES" in values
+    if admin_routes_touched:
+        values["ADMIN_NOTIFICATION_ROUTES"] = _ensure_default_route_row(normalized_admin_routes)
+
+    user_routes_touched = "USER_NOTIFICATION_ROUTES" in values
+    if user_routes_touched:
+        values["USER_NOTIFICATION_ROUTES"] = _ensure_default_route_row(normalized_user_routes)
+
+    return {"error": False, "values": values}
+
+
+def _test_admin_notification_action(current_values: dict[str, Any]) -> dict[str, Any]:
+    persisted = load_config_file("notifications")
+    effective: dict[str, Any] = dict(persisted)
+    if isinstance(current_values, dict):
+        effective.update(current_values)
+
+    routes_input = effective.get("ADMIN_NOTIFICATION_ROUTES", [])
+    return build_notification_test_result(routes_input, scope_label="global")
+
+
+register_on_save("notifications", _on_save_notifications)
+
+
+@register_settings("notifications", "Notifications", icon="bell", order=7)
+def notifications_settings():
+    """Global notifications settings."""
+    return [
+        HeadingField(
+            key="notifications_heading",
+            title="Global Notifications",
+            description=(
+                "Global notifications send selected events for all users to configured routes. "
+                "Users can manage personal notifications in User Preferences."
+            ),
+        ),
+        TableField(
+            key="ADMIN_NOTIFICATION_ROUTES",
+            label="",
+            description=(
+                "Create one route per URL. Start with All, then add event-specific routes "
+                "for targeted delivery. Need format examples? "
+                "[View Apprise URL formats](https://appriseit.com/services/)."
+            ),
+            columns=[
+                {
+                    "key": "event",
+                    "label": "Event",
+                    "type": "multiselect",
+                    "options": _ROUTE_EVENT_OPTIONS,
+                    "defaultValue": [_ROUTE_EVENT_ALL],
+                    "placeholder": "Select events...",
+                },
+                {
+                    "key": "url",
+                    "label": "Notification URL",
+                    "type": "text",
+                    "placeholder": "e.g. ntfys://ntfy.sh/shelfmark",
+                },
+            ],
+            default=[dict(row) for row in _DEFAULT_ROUTE_ROWS],
+            add_label="Add Route",
+            empty_message="No routes configured.",
+        ),
+        ActionButton(
+            key="test_admin_notification",
+            label="Test Notification",
+            description="Send a test notification to all configured global route URLs.",
+            style="primary",
+            callback=_test_admin_notification_action,
+        ),
+        TableField(
+            key="USER_NOTIFICATION_ROUTES",
+            label="",
+            description=(
+                "Create one route per URL. Start with All, then add event-specific routes "
+                "for targeted delivery. Need format examples? "
+                "[View Apprise URL formats](https://appriseit.com/services/)."
+            ),
+            columns=[
+                {
+                    "key": "event",
+                    "label": "Event",
+                    "type": "multiselect",
+                    "options": _ROUTE_EVENT_OPTIONS,
+                    "defaultValue": [_ROUTE_EVENT_ALL],
+                    "placeholder": "Select events...",
+                },
+                {
+                    "key": "url",
+                    "label": "Notification URL",
+                    "type": "text",
+                    "placeholder": "e.g. ntfys://ntfy.sh/username-topic",
+                },
+            ],
+            default=[dict(row) for row in _DEFAULT_ROUTE_ROWS],
+            add_label="Add Route",
+            empty_message="No routes configured.",
+            user_overridable=True,
+            hidden_in_ui=True,
+        ),
+    ]

shelfmark/config/security.py

@@ -0,0 +1,245 @@
+"""Authentication settings registration."""
+
+from typing import Any, Dict, Callable
+
+from shelfmark.config.migrations import migrate_security_settings
+from shelfmark.config.security_handlers import (
+    on_save_security,
+    test_oidc_connection,
+)
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    register_settings,
+    register_on_save,
+    load_config_file,
+    TextField,
+    SelectField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+    TagListField,
+)
+from shelfmark.core.user_db import sync_builtin_admin_user
+
+logger = setup_logger(__name__)
+
+
+def _auth_condition(auth_method: str) -> dict[str, str]:
+    return {"field": "AUTH_METHOD", "value": auth_method}
+
+
+def _ui_field(factory: Callable[..., Any], **kwargs: Any) -> Any:
+    return factory(env_supported=False, **kwargs)
+
+
+def _auth_ui_field(factory: Callable[..., Any], auth_method: str, **kwargs: Any) -> Any:
+    return _ui_field(factory, show_when=_auth_condition(auth_method), **kwargs)
+
+
+def _migrate_security_settings() -> None:
+    from shelfmark.core.settings_registry import (
+        _get_config_file_path,
+        _ensure_config_dir,
+        save_config_file,
+    )
+
+    migrate_security_settings(
+        load_security_config=lambda: load_config_file("security"),
+        load_users_config=lambda: load_config_file("users"),
+        save_users_config=lambda values: save_config_file("users", values),
+        ensure_config_dir=lambda: _ensure_config_dir("security"),
+        get_config_path=lambda: _get_config_file_path("security"),
+        sync_builtin_admin_user=sync_builtin_admin_user,
+        logger=logger,
+    )
+
+
+
+def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
+    return on_save_security(values)
+
+
+def _test_oidc_connection() -> Dict[str, Any]:
+    return test_oidc_connection(
+        load_security_config=lambda: load_config_file("security"),
+        logger=logger,
+    )
+
+
+@register_settings("security", "Security", icon="shield", order=5)
+def security_settings():
+    """Security and authentication settings."""
+    from shelfmark.config.env import CWA_DB_PATH
+
+    cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
+
+    auth_method_options = [
+        {"label": "No Authentication", "value": "none"},
+        {"label": "Local", "value": "builtin"},
+        {"label": "Proxy Authentication", "value": "proxy"},
+        {"label": "OIDC (OpenID Connect)", "value": "oidc"},
+    ]
+    if cwa_db_available:
+        auth_method_options.append({"label": "Calibre-Web Database", "value": "cwa"})
+
+    auth_method_description = "Select the authentication method for accessing Shelfmark."
+    if not cwa_db_available:
+        auth_method_description += " Calibre-Web database option requires mounting your Calibre-Web app.db to /auth/app.db."
+
+    fields = [
+        SelectField(
+            key="AUTH_METHOD",
+            label="Authentication Method",
+            description=auth_method_description,
+            options=auth_method_options,
+            default="none",
+            env_supported=False,
+        ),
+        ActionButton(
+            key="open_users_tab",
+            label="Go to Users",
+            description="Configure local users and admin access in the Users tab.",
+            style="primary",
+            show_when=_auth_condition("builtin"),
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_USER_HEADER",
+            label="Proxy Auth User Header",
+            description="The HTTP header your proxy uses to pass the authenticated username.",
+            placeholder="e.g. X-Auth-User",
+            default="X-Auth-User",
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_LOGOUT_URL",
+            label="Proxy Auth Logout URL",
+            description="The URL to redirect users to for logging out. Leave empty to disable logout functionality.",
+            placeholder="https://myauth.example.com/logout",
+            default="",
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_ADMIN_GROUP_HEADER",
+            label="Proxy Auth Admin Group Header",
+            description="Optional: header your proxy uses to pass user groups/roles.",
+            placeholder="e.g. X-Auth-Groups",
+            default="X-Auth-Groups",
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_ADMIN_GROUP_NAME",
+            label="Proxy Auth Admin Group",
+            description="Optional: users in this group are treated as admins. Leave blank to skip group-based admin detection.",
+            placeholder="e.g. admins",
+            default="",
+        ),
+    ]
+
+    oidc_specs = [
+        (
+            TextField,
+            {
+                "key": "OIDC_DISCOVERY_URL",
+                "label": "Discovery URL",
+                "description": "OpenID Connect discovery endpoint URL. Usually ends with /.well-known/openid-configuration.",
+                "placeholder": "https://auth.example.com/.well-known/openid-configuration",
+                "required": True,
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_CLIENT_ID",
+                "label": "Client ID",
+                "description": "OAuth2 client ID from your identity provider.",
+                "placeholder": "shelfmark",
+                "required": True,
+            },
+        ),
+        (
+            PasswordField,
+            {
+                "key": "OIDC_CLIENT_SECRET",
+                "label": "Client Secret",
+                "description": "OAuth2 client secret from your identity provider.",
+                "required": True,
+            },
+        ),
+        (
+            TagListField,
+            {
+                "key": "OIDC_SCOPES",
+                "label": "Scopes",
+                "description": "OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization.",
+                "default": ["openid", "email", "profile"],
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_GROUP_CLAIM",
+                "label": "Group Claim Name",
+                "description": "The name of the claim in the ID token that contains user groups.",
+                "placeholder": "groups",
+                "default": "groups",
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_ADMIN_GROUP",
+                "label": "Admin Group Name",
+                "description": "Users in this group will be given admin access (if enabled below). Leave empty to use database roles only.",
+                "placeholder": "shelfmark-admins",
+                "default": "",
+            },
+        ),
+        (
+            CheckboxField,
+            {
+                "key": "OIDC_USE_ADMIN_GROUP",
+                "label": "Use Admin Group for Authorization",
+                "description": "When enabled, users in the Admin Group are granted admin access. When disabled, admin access is determined solely by database roles.",
+                "default": True,
+            },
+        ),
+        (
+            CheckboxField,
+            {
+                "key": "OIDC_AUTO_PROVISION",
+                "label": "Auto-Provision Users",
+                "description": "Automatically create a user account on first OIDC login. When disabled, users must be pre-created by an admin.",
+                "default": True,
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_BUTTON_LABEL",
+                "label": "Login Button Label",
+                "description": "Custom label for the OIDC sign-in button on the login page.",
+                "placeholder": "Sign in with OIDC",
+                "default": "",
+            },
+        ),
+    ]
+    fields.extend(_auth_ui_field(factory, "oidc", **spec) for factory, spec in oidc_specs)
+    fields.append(
+        ActionButton(
+            key="test_oidc",
+            label="Test Connection",
+            description="Fetch the OIDC discovery document and validate configuration.",
+            style="primary",
+            callback=_test_oidc_connection,
+            show_when=_auth_condition("oidc"),
+        )
+    )
+    return fields
+
+
+register_on_save("security", _on_save_security)

shelfmark/config/security_handlers.py

@@ -0,0 +1,72 @@
+"""Operational handlers for security settings (save/actions)."""
+
+import os
+from typing import Any, Callable
+
+from shelfmark.core.utils import normalize_http_url
+from shelfmark.core.user_db import UserDB
+
+
+_OIDC_LOCKOUT_MESSAGE = "Create a local admin account first (Users tab) before enabling OIDC. This ensures you can still log in with a password if SSO is unavailable."
+
+
+def _has_local_password_admin() -> bool:
+    root = os.environ.get("CONFIG_DIR", "/config")
+    user_db = UserDB(os.path.join(root, "users.db"))
+    user_db.initialize()
+    return any(user.get("password_hash") and user.get("role") == "admin" for user in user_db.list_users())
+
+
+def on_save_security(
+    values: dict[str, Any],
+) -> dict[str, Any]:
+    """Validate security values before persistence."""
+    normalized_values = values.copy()
+
+    discovery_url = normalized_values.get("OIDC_DISCOVERY_URL")
+    if discovery_url is not None:
+        normalized_values["OIDC_DISCOVERY_URL"] = normalize_http_url(
+            str(discovery_url),
+            default_scheme="https",
+        )
+
+    proxy_logout_url = normalized_values.get("PROXY_AUTH_LOGOUT_URL")
+    if proxy_logout_url is not None:
+        normalized_values["PROXY_AUTH_LOGOUT_URL"] = normalize_http_url(
+            str(proxy_logout_url),
+            default_scheme="https",
+            strip_trailing_slash=False,
+        )
+
+    if normalized_values.get("AUTH_METHOD") == "oidc" and not _has_local_password_admin():
+        return {"error": True, "message": _OIDC_LOCKOUT_MESSAGE, "values": normalized_values}
+
+    return {"error": False, "values": normalized_values}
+
+
+def test_oidc_connection(
+    *,
+    load_security_config: Callable[[], dict[str, Any]],
+    logger: Any,
+) -> dict[str, Any]:
+    """Fetch and validate the configured OIDC discovery document."""
+    import requests
+
+    try:
+        discovery_url = load_security_config().get("OIDC_DISCOVERY_URL", "")
+        if not discovery_url:
+            return {"success": False, "message": "Discovery URL is not configured."}
+
+        response = requests.get(discovery_url, timeout=10)
+        response.raise_for_status()
+        document = response.json()
+
+        required_fields = ["issuer", "authorization_endpoint", "token_endpoint"]
+        missing_fields = [field for field in required_fields if field not in document]
+        if missing_fields:
+            return {"success": False, "message": f"Discovery document missing fields: {', '.join(missing_fields)}"}
+
+        return {"success": True, "message": f"Connected to {document['issuer']}"}
+    except Exception as exc:
+        logger.error(f"OIDC connection test failed: {exc}")
+        return {"success": False, "message": f"Connection failed: {str(exc)}"}

shelfmark/config/users_settings.py

@@ -0,0 +1,320 @@
+"""Users settings tab registration.
+
+This registers a 'users' tab in the settings sidebar.
+The actual user management is handled by a custom frontend component
+that talks to /api/admin/users endpoints.
+"""
+
+from shelfmark.core.settings_registry import (
+    CheckboxField,
+    CustomComponentField,
+    HeadingField,
+    MultiSelectField,
+    NumberField,
+    SelectField,
+    TableField,
+    register_on_save,
+    register_settings,
+)
+from shelfmark.core.request_policy import (
+    get_source_content_type_capabilities,
+    parse_policy_mode,
+    validate_policy_rules,
+)
+
+
+_REQUEST_DEFAULT_MODE_OPTIONS = [
+    {
+        "value": "download",
+        "label": "Download",
+        "description": "Everything can be downloaded directly.",
+    },
+    {
+        "value": "request_release",
+        "label": "Request Release",
+        "description": "Users must request a specific release.",
+    },
+    {
+        "value": "request_book",
+        "label": "Request Book",
+        "description": "Users request a book, admin picks the release.",
+    },
+    {
+        "value": "blocked",
+        "label": "Blocked",
+        "description": "No downloads or requests allowed.",
+    },
+]
+
+_REQUEST_MATRIX_MODE_OPTIONS = [
+    option for option in _REQUEST_DEFAULT_MODE_OPTIONS if option["value"] != "request_book"
+]
+
+_SELF_SETTINGS_SECTION_OPTIONS = [
+    {
+        "value": "delivery",
+        "label": "Delivery Preferences",
+        "description": "Show personal delivery output and destination settings.",
+    },
+    {
+        "value": "notifications",
+        "label": "Notifications",
+        "description": "Show personal notification route settings.",
+    },
+]
+_SELF_SETTINGS_SECTION_VALUES = {option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS}
+_SELF_SETTINGS_SECTION_DEFAULTS = [option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS]
+
+_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE = {
+    "builtin": (
+        "Create and manage user accounts directly. Passwords are stored locally and users sign in "
+        "with their username and password."
+    ),
+    "oidc": (
+        "Users sign in through your identity provider. New accounts can be created automatically on "
+        "first login when auto-provisioning is enabled, or you can pre-create users here and they\u2019ll "
+        "be linked by email on first sign-in."
+    ),
+    "proxy": (
+        "Users are authenticated by your reverse proxy. Accounts are automatically created on first "
+        "sign-in. If a local user with a matching username already exists, it will be linked instead."
+    ),
+    "cwa": (
+        "User accounts are synced from your Calibre-Web database. Users are matched by email, and new "
+        "accounts are created here when new CWA users are found."
+    ),
+    "none": "Authentication is disabled. Anyone can access Shelfmark without signing in.",
+    "default": "Authentication is disabled. Anyone can access Shelfmark without signing in.",
+}
+
+
+def _get_request_source_options():
+    """Build request-policy source options from registered release sources."""
+    from shelfmark.release_sources import list_available_sources
+
+    options = []
+    for source in list_available_sources():
+        options.append(
+            {
+                "value": source["name"],
+                "label": source["display_name"],
+            }
+        )
+    return options
+
+
+def _get_request_policy_rule_columns():
+    source_capabilities = get_source_content_type_capabilities()
+    content_type_options = []
+
+    for source_name, supported_types in source_capabilities.items():
+        normalized_types = [t for t in ("ebook", "audiobook") if t in supported_types]
+        for content_type in normalized_types:
+            content_type_options.append(
+                {
+                    "value": content_type,
+                    "label": "Ebook" if content_type == "ebook" else "Audiobook",
+                    "childOf": source_name,
+                }
+            )
+
+    return [
+        {
+            "key": "source",
+            "label": "Source",
+            "type": "select",
+            "options": _get_request_source_options(),
+            "defaultValue": "",
+            "placeholder": "Select source...",
+        },
+        {
+            "key": "content_type",
+            "label": "Content Type",
+            "type": "select",
+            "options": content_type_options,
+            "defaultValue": "",
+            "placeholder": "Select content type...",
+            "filterByField": "source",
+        },
+        {
+            "key": "mode",
+            "label": "Mode",
+            "type": "select",
+            "options": _REQUEST_MATRIX_MODE_OPTIONS,
+            "defaultValue": "",
+            "placeholder": "Select mode...",
+        },
+    ]
+
+
+def _on_save_users(values):
+    """Validate users/request-policy settings before persistence."""
+    if "VISIBLE_SELF_SETTINGS_SECTIONS" in values:
+        raw_sections = values["VISIBLE_SELF_SETTINGS_SECTIONS"]
+        if raw_sections is None:
+            candidate_sections: list[str] = []
+        elif isinstance(raw_sections, str):
+            candidate_sections = [s.strip() for s in raw_sections.split(",") if s.strip()]
+        elif isinstance(raw_sections, (list, tuple, set)):
+            candidate_sections = [str(section).strip() for section in raw_sections if str(section).strip()]
+        else:
+            return {
+                "error": True,
+                "message": "VISIBLE_SELF_SETTINGS_SECTIONS must be a list of section identifiers",
+                "values": values,
+            }
+
+        normalized_sections: list[str] = []
+        for section in candidate_sections:
+            if section not in _SELF_SETTINGS_SECTION_VALUES:
+                allowed = ", ".join(sorted(_SELF_SETTINGS_SECTION_VALUES))
+                return {
+                    "error": True,
+                    "message": (
+                        "VISIBLE_SELF_SETTINGS_SECTIONS contains an unsupported section "
+                        f"'{section}'. Supported values: {allowed}"
+                    ),
+                    "values": values,
+                }
+            if section not in normalized_sections:
+                normalized_sections.append(section)
+
+        values["VISIBLE_SELF_SETTINGS_SECTIONS"] = normalized_sections
+
+    if "REQUEST_POLICY_DEFAULT_EBOOK" in values:
+        if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_EBOOK"]) is None:
+            return {
+                "error": True,
+                "message": "REQUEST_POLICY_DEFAULT_EBOOK must be a valid policy mode",
+                "values": values,
+            }
+
+    if "REQUEST_POLICY_DEFAULT_AUDIOBOOK" in values:
+        if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_AUDIOBOOK"]) is None:
+            return {
+                "error": True,
+                "message": "REQUEST_POLICY_DEFAULT_AUDIOBOOK must be a valid policy mode",
+                "values": values,
+            }
+
+    if "REQUEST_POLICY_RULES" in values:
+        normalized_rules, errors = validate_policy_rules(values["REQUEST_POLICY_RULES"])
+        if errors:
+            return {
+                "error": True,
+                "message": "; ".join(errors),
+                "values": values,
+            }
+        values["REQUEST_POLICY_RULES"] = normalized_rules
+
+    return {"error": False, "values": values}
+
+
+register_on_save("users", _on_save_users)
+
+
+@register_settings("users", "Users & Requests", icon="users", order=6)
+def users_settings():
+    """User management tab - rendered as a custom component on the frontend."""
+    return [
+        HeadingField(
+            key="users_heading",
+            title="Users",
+            description=_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE["default"],
+            description_by_auth_mode=_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE,
+        ),
+        CustomComponentField(
+            key="users_management",
+            component="users_management",
+        ),
+        MultiSelectField(
+            key="VISIBLE_SELF_SETTINGS_SECTIONS",
+            label="Visible Self-Settings Sections",
+            description=(
+                "Choose which personal settings sections are shown in My Account for non-admin users."
+            ),
+            options=_SELF_SETTINGS_SECTION_OPTIONS,
+            default=_SELF_SETTINGS_SECTION_DEFAULTS,
+            variant="dropdown",
+            env_supported=False,
+        ),
+        HeadingField(
+            key="requests_heading",
+            title="Requests",
+            description=(
+                "Choose what users can download directly and what needs approval first."
+            ),
+        ),
+        CheckboxField(
+            key="REQUESTS_ENABLED",
+            label="Enable Requests",
+            description=(
+                "Turn this off to let everyone download directly without needing approval."
+            ),
+            default=False,
+            user_overridable=True,
+        ),
+        CustomComponentField(
+            key="request_policy_editor",
+            component="request_policy_grid",
+            label="Request Rules",
+            description=(
+                "Fine-tune access per source. Source rules can only be the same or more restrictive than the default above."
+            ),
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+            wrap_in_field_wrapper=True,
+            value_fields=[
+                SelectField(
+                    key="REQUEST_POLICY_DEFAULT_EBOOK",
+                    label="Default Ebook Mode",
+                    description=(
+                        "Sets the baseline for all ebook sources."
+                    ),
+                    options=_REQUEST_DEFAULT_MODE_OPTIONS,
+                    default="download",
+                    user_overridable=True,
+                ),
+                SelectField(
+                    key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
+                    label="Default Audiobook Mode",
+                    description=(
+                        "Sets the baseline for all audiobook sources."
+                    ),
+                    options=_REQUEST_DEFAULT_MODE_OPTIONS,
+                    default="download",
+                    user_overridable=True,
+                ),
+                TableField(
+                    key="REQUEST_POLICY_RULES",
+                    label="Request Rules",
+                    description=(
+                        "Fine-tune access per source. Source rules can only be the same or more restrictive than the default above."
+                    ),
+                    columns=_get_request_policy_rule_columns,
+                    default=[],
+                    add_label="Add Rule",
+                    empty_message="No request policy rules configured.",
+                    env_supported=False,
+                    user_overridable=True,
+                ),
+            ],
+        ),
+        NumberField(
+            key="MAX_PENDING_REQUESTS_PER_USER",
+            label="Max pending requests per user",
+            description="How many open requests a user can have at a time.",
+            default=20,
+            min_value=1,
+            max_value=1000,
+            user_overridable=True,
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+        ),
+        CheckboxField(
+            key="REQUESTS_ALLOW_NOTES",
+            label="Allow notes on requests",
+            description="Let users add a note when they submit a request.",
+            default=True,
+            user_overridable=True,
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+        ),
+    ]

shelfmark/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core module - shared models, queue, and utilities."""
+
+from shelfmark.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
+from shelfmark.core.queue import BookQueue, book_queue
+from shelfmark.core.logger import setup_logger

shelfmark/core/activity_routes.py

@@ -0,0 +1,486 @@
+"""Activity API routes (snapshot, dismiss, history)."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from flask import Flask, jsonify, request, session
+
+from shelfmark.core.activity_service import ActivityService
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+
+
+def _require_authenticated(resolve_auth_mode: Callable[[], str]):
+    auth_mode = resolve_auth_mode()
+    if auth_mode == "none":
+        return None
+    if "user_id" not in session:
+        return jsonify({"error": "Unauthorized"}), 401
+    return None
+
+
+def _resolve_db_user_id(require_in_auth_mode: bool = True):
+    raw_db_user_id = session.get("db_user_id")
+    if raw_db_user_id is None:
+        if not require_in_auth_mode:
+            return None, None
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+    try:
+        return int(raw_db_user_id), None
+    except (TypeError, ValueError):
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+
+
+def _emit_activity_event(ws_manager: Any | None, *, room: str, payload: dict[str, Any]) -> None:
+    if ws_manager is None:
+        return
+    try:
+        socketio = getattr(ws_manager, "socketio", None)
+        is_enabled = getattr(ws_manager, "is_enabled", None)
+        if socketio is None or not callable(is_enabled) or not is_enabled():
+            return
+        socketio.emit("activity_update", payload, to=room)
+    except Exception as exc:
+        logger.warning("Failed to emit activity_update event: %s", exc)
+
+
+def _list_visible_requests(user_db: UserDB, *, is_admin: bool, db_user_id: int | None) -> list[dict[str, Any]]:
+    if is_admin:
+        request_rows = user_db.list_requests()
+        user_cache: dict[int, str] = {}
+        for row in request_rows:
+            requester_id = row["user_id"]
+            if requester_id not in user_cache:
+                requester = user_db.get_user(user_id=requester_id)
+                user_cache[requester_id] = requester.get("username", "") if requester else ""
+            row["username"] = user_cache[requester_id]
+        return request_rows
+
+    if db_user_id is None:
+        return []
+    return user_db.list_requests(user_id=db_user_id)
+
+
+def _parse_download_item_key(item_key: str) -> str | None:
+    if not isinstance(item_key, str) or not item_key.startswith("download:"):
+        return None
+    task_id = item_key.split(":", 1)[1].strip()
+    return task_id or None
+
+
+def _parse_request_item_key(item_key: str) -> int | None:
+    if not isinstance(item_key, str) or not item_key.startswith("request:"):
+        return None
+    raw_id = item_key.split(":", 1)[1].strip()
+    try:
+        parsed = int(raw_id)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _task_id_from_download_item_key(item_key: str) -> str | None:
+    task_id = _parse_download_item_key(item_key)
+    if task_id is None:
+        return None
+    return task_id
+
+
+def _merge_terminal_snapshot_backfill(
+    *,
+    status: dict[str, dict[str, Any]],
+    terminal_rows: list[dict[str, Any]],
+) -> None:
+    existing_task_ids: set[str] = set()
+    for bucket_key in ("queued", "resolving", "locating", "downloading", "complete", "error", "cancelled"):
+        bucket = status.get(bucket_key)
+        if not isinstance(bucket, dict):
+            continue
+        existing_task_ids.update(str(task_id) for task_id in bucket.keys())
+
+    for row in terminal_rows:
+        item_key = row.get("item_key")
+        if not isinstance(item_key, str):
+            continue
+        task_id = _task_id_from_download_item_key(item_key)
+        if not task_id or task_id in existing_task_ids:
+            continue
+
+        final_status = row.get("final_status")
+        if final_status not in {"complete", "error", "cancelled"}:
+            continue
+
+        snapshot = row.get("snapshot")
+        if not isinstance(snapshot, dict):
+            continue
+        raw_download = snapshot.get("download")
+        if not isinstance(raw_download, dict):
+            continue
+
+        download_payload = dict(raw_download)
+        if not isinstance(download_payload.get("id"), str):
+            download_payload["id"] = task_id
+
+        if final_status not in status or not isinstance(status.get(final_status), dict):
+            status[final_status] = {}
+        status[final_status][task_id] = download_payload
+        existing_task_ids.add(task_id)
+
+
+def _collect_active_download_item_keys(status: dict[str, dict[str, Any]]) -> set[str]:
+    active_keys: set[str] = set()
+    for bucket_key in ("queued", "resolving", "locating", "downloading"):
+        bucket = status.get(bucket_key)
+        if not isinstance(bucket, dict):
+            continue
+        for task_id in bucket.keys():
+            normalized_task_id = str(task_id).strip()
+            if not normalized_task_id:
+                continue
+            active_keys.add(f"download:{normalized_task_id}")
+    return active_keys
+
+
+def _extract_request_source_id(row: dict[str, Any]) -> str | None:
+    release_data = row.get("release_data")
+    if not isinstance(release_data, dict):
+        return None
+    source_id = release_data.get("source_id")
+    if not isinstance(source_id, str):
+        return None
+    normalized = source_id.strip()
+    return normalized or None
+
+
+def _request_terminal_status(row: dict[str, Any]) -> str | None:
+    request_status = row.get("status")
+    if request_status == "pending":
+        return None
+    if request_status == "rejected":
+        return "rejected"
+    if request_status == "cancelled":
+        return "cancelled"
+    if request_status != "fulfilled":
+        return None
+
+    delivery_state = str(row.get("delivery_state") or "").strip().lower()
+    if delivery_state in {"error", "cancelled"}:
+        return delivery_state
+    return "complete"
+
+
+def _minimal_request_snapshot(request_row: dict[str, Any], request_id: int) -> dict[str, Any]:
+    book_data = request_row.get("book_data")
+    release_data = request_row.get("release_data")
+    if not isinstance(book_data, dict):
+        book_data = {}
+    if not isinstance(release_data, dict):
+        release_data = {}
+
+    minimal_request = {
+        "id": request_id,
+        "user_id": request_row.get("user_id"),
+        "status": request_row.get("status"),
+        "request_level": request_row.get("request_level"),
+        "delivery_state": request_row.get("delivery_state"),
+        "book_data": book_data,
+        "release_data": release_data,
+        "note": request_row.get("note"),
+        "admin_note": request_row.get("admin_note"),
+        "created_at": request_row.get("created_at"),
+        "updated_at": request_row.get("updated_at"),
+    }
+    username = request_row.get("username")
+    if isinstance(username, str):
+        minimal_request["username"] = username
+    return {"kind": "request", "request": minimal_request}
+
+
+def _get_existing_activity_log_id_for_item(
+    *,
+    activity_service: ActivityService,
+    user_db: UserDB,
+    item_type: str,
+    item_key: str,
+) -> int | None:
+    if item_type not in {"request", "download"}:
+        return None
+    if not isinstance(item_key, str) or not item_key.strip():
+        return None
+
+    existing_log_id = activity_service.get_latest_activity_log_id(
+        item_type=item_type,
+        item_key=item_key,
+    )
+    if existing_log_id is not None or item_type != "request":
+        return existing_log_id
+
+    request_id = _parse_request_item_key(item_key)
+    if request_id is None:
+        return None
+    row = user_db.get_request(request_id)
+    if row is None:
+        return None
+
+    final_status = _request_terminal_status(row)
+    if final_status is None:
+        return None
+
+    source_id = _extract_request_source_id(row)
+    payload = activity_service.record_terminal_snapshot(
+        user_id=row.get("user_id"),
+        item_type="request",
+        item_key=item_key,
+        origin="request",
+        final_status=final_status,
+        snapshot=_minimal_request_snapshot(row, request_id),
+        request_id=request_id,
+        source_id=source_id,
+    )
+    return int(payload["id"])
+
+
+def register_activity_routes(
+    app: Flask,
+    user_db: UserDB,
+    *,
+    activity_service: ActivityService,
+    resolve_auth_mode: Callable[[], str],
+    resolve_status_scope: Callable[[], tuple[bool, int | None, bool]],
+    queue_status: Callable[..., dict[str, dict[str, Any]]],
+    sync_request_delivery_states: Callable[..., list[dict[str, Any]]],
+    emit_request_updates: Callable[[list[dict[str, Any]]], None],
+    ws_manager: Any | None = None,
+) -> None:
+    """Register activity routes."""
+
+    @app.route("/api/activity/snapshot", methods=["GET"])
+    def api_activity_snapshot():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        is_admin, db_user_id, can_access_status = resolve_status_scope()
+        if not can_access_status:
+            return (
+                jsonify(
+                    {
+                        "error": "User identity unavailable for activity workflow",
+                        "code": "user_identity_unavailable",
+                    }
+                ),
+                403,
+            )
+
+        viewer_db_user_id, _ = _resolve_db_user_id(require_in_auth_mode=False)
+        scoped_user_id = None if is_admin else db_user_id
+        status = queue_status(user_id=scoped_user_id)
+        updated_requests = sync_request_delivery_states(
+            user_db,
+            queue_status=status,
+            user_id=scoped_user_id,
+        )
+        emit_request_updates(updated_requests)
+        request_rows = _list_visible_requests(user_db, is_admin=is_admin, db_user_id=db_user_id)
+
+        if not is_admin and db_user_id is not None:
+            try:
+                terminal_rows = activity_service.get_undismissed_terminal_downloads(db_user_id, limit=200)
+                _merge_terminal_snapshot_backfill(status=status, terminal_rows=terminal_rows)
+            except Exception as exc:
+                logger.warning("Failed to merge terminal snapshot backfill rows: %s", exc)
+
+        if viewer_db_user_id is not None:
+            active_download_keys = _collect_active_download_item_keys(status)
+            if active_download_keys:
+                try:
+                    activity_service.clear_dismissals_for_item_keys(
+                        user_id=viewer_db_user_id,
+                        item_type="download",
+                        item_keys=active_download_keys,
+                    )
+                except Exception as exc:
+                    logger.warning("Failed to clear stale download dismissals for active tasks: %s", exc)
+
+        dismissed: list[dict[str, str]] = []
+        # Admins can view unscoped queue status, but dismissals remain per-viewer.
+        if viewer_db_user_id is not None:
+            dismissed = activity_service.get_dismissal_set(viewer_db_user_id)
+
+        return jsonify(
+            {
+                "status": status,
+                "requests": request_rows,
+                "dismissed": dismissed,
+            }
+        )
+
+    @app.route("/api/activity/dismiss", methods=["POST"])
+    def api_activity_dismiss():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        data = request.get_json(silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": "Invalid payload"}), 400
+
+        activity_log_id = data.get("activity_log_id")
+        if activity_log_id is None:
+            try:
+                activity_log_id = _get_existing_activity_log_id_for_item(
+                    activity_service=activity_service,
+                    user_db=user_db,
+                    item_type=data.get("item_type"),
+                    item_key=data.get("item_key"),
+                )
+            except Exception as exc:
+                logger.warning("Failed to resolve activity snapshot id for dismiss payload: %s", exc)
+                activity_log_id = None
+
+        try:
+            dismissal = activity_service.dismiss_item(
+                user_id=db_user_id,
+                item_type=data.get("item_type"),
+                item_key=data.get("item_key"),
+                activity_log_id=activity_log_id,
+            )
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+
+        _emit_activity_event(
+            ws_manager,
+            room=f"user_{db_user_id}",
+            payload={
+                "kind": "dismiss",
+                "user_id": db_user_id,
+                "item_type": dismissal["item_type"],
+                "item_key": dismissal["item_key"],
+            },
+        )
+
+        return jsonify({"status": "dismissed", "item": dismissal})
+
+    @app.route("/api/activity/dismiss-many", methods=["POST"])
+    def api_activity_dismiss_many():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        data = request.get_json(silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": "Invalid payload"}), 400
+        items = data.get("items")
+        if not isinstance(items, list):
+            return jsonify({"error": "items must be an array"}), 400
+
+        normalized_items: list[dict[str, Any]] = []
+        for item in items:
+            if not isinstance(item, dict):
+                return jsonify({"error": "items must contain objects"}), 400
+
+            activity_log_id = item.get("activity_log_id")
+            if activity_log_id is None:
+                try:
+                    activity_log_id = _get_existing_activity_log_id_for_item(
+                        activity_service=activity_service,
+                        user_db=user_db,
+                        item_type=item.get("item_type"),
+                        item_key=item.get("item_key"),
+                    )
+                except Exception as exc:
+                    logger.warning("Failed to resolve activity snapshot id for dismiss-many item: %s", exc)
+                    activity_log_id = None
+
+            normalized_payload = {
+                "item_type": item.get("item_type"),
+                "item_key": item.get("item_key"),
+            }
+            if activity_log_id is not None:
+                normalized_payload["activity_log_id"] = activity_log_id
+            normalized_items.append(normalized_payload)
+
+        try:
+            dismissed_count = activity_service.dismiss_many(user_id=db_user_id, items=normalized_items)
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+
+        _emit_activity_event(
+            ws_manager,
+            room=f"user_{db_user_id}",
+            payload={
+                "kind": "dismiss_many",
+                "user_id": db_user_id,
+                "count": dismissed_count,
+            },
+        )
+
+        return jsonify({"status": "dismissed", "count": dismissed_count})
+
+    @app.route("/api/activity/history", methods=["GET"])
+    def api_activity_history():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        limit = request.args.get("limit", type=int, default=50) or 50
+        offset = request.args.get("offset", type=int, default=0) or 0
+
+        try:
+            history = activity_service.get_history(db_user_id, limit=limit, offset=offset)
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+        return jsonify(history)
+
+    @app.route("/api/activity/history", methods=["DELETE"])
+    def api_activity_history_clear():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        deleted_count = activity_service.clear_history(db_user_id)
+        _emit_activity_event(
+            ws_manager,
+            room=f"user_{db_user_id}",
+            payload={
+                "kind": "history_cleared",
+                "user_id": db_user_id,
+                "count": deleted_count,
+            },
+        )
+        return jsonify({"status": "cleared", "deleted_count": deleted_count})

shelfmark/core/activity_service.py

@@ -0,0 +1,618 @@
+"""Persistence helpers for Activity dismissals and terminal snapshots."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import sqlite3
+from typing import Any, Iterable
+
+
+VALID_ITEM_TYPES = frozenset({"download", "request"})
+VALID_ORIGINS = frozenset({"direct", "request", "requested"})
+VALID_FINAL_STATUSES = frozenset({"complete", "error", "cancelled", "rejected"})
+
+
+def _now_timestamp() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _normalize_item_type(item_type: Any) -> str:
+    if not isinstance(item_type, str):
+        raise ValueError("item_type must be a string")
+    normalized = item_type.strip().lower()
+    if normalized not in VALID_ITEM_TYPES:
+        raise ValueError("item_type must be one of: download, request")
+    return normalized
+
+
+def _normalize_item_key(item_key: Any) -> str:
+    if not isinstance(item_key, str):
+        raise ValueError("item_key must be a string")
+    normalized = item_key.strip()
+    if not normalized:
+        raise ValueError("item_key must not be empty")
+    return normalized
+
+
+def _normalize_origin(origin: Any) -> str:
+    if not isinstance(origin, str):
+        raise ValueError("origin must be a string")
+    normalized = origin.strip().lower()
+    if normalized not in VALID_ORIGINS:
+        raise ValueError("origin must be one of: direct, request, requested")
+    return normalized
+
+
+def _normalize_final_status(final_status: Any) -> str:
+    if not isinstance(final_status, str):
+        raise ValueError("final_status must be a string")
+    normalized = final_status.strip().lower()
+    if normalized not in VALID_FINAL_STATUSES:
+        raise ValueError("final_status must be one of: complete, error, cancelled, rejected")
+    return normalized
+
+
+def build_item_key(item_type: str, raw_id: Any) -> str:
+    """Build a stable item key used by dismiss/history APIs."""
+    normalized_type = _normalize_item_type(item_type)
+    if normalized_type == "request":
+        try:
+            request_id = int(raw_id)
+        except (TypeError, ValueError) as exc:
+            raise ValueError("request item IDs must be integers") from exc
+        if request_id < 1:
+            raise ValueError("request item IDs must be positive integers")
+        return f"request:{request_id}"
+
+    if not isinstance(raw_id, str):
+        raise ValueError("download item IDs must be strings")
+    task_id = raw_id.strip()
+    if not task_id:
+        raise ValueError("download item IDs must not be empty")
+    return f"download:{task_id}"
+
+
+def build_request_item_key(request_id: int) -> str:
+    """Build a request item key."""
+    return build_item_key("request", request_id)
+
+
+def build_download_item_key(task_id: str) -> str:
+    """Build a download item key."""
+    return build_item_key("download", task_id)
+
+
+def _parse_request_id_from_item_key(item_key: Any) -> int | None:
+    if not isinstance(item_key, str) or not item_key.startswith("request:"):
+        return None
+    raw_value = item_key.split(":", 1)[1].strip()
+    try:
+        parsed = int(raw_value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _request_final_status(request_status: Any, delivery_state: Any) -> str | None:
+    status = str(request_status or "").strip().lower()
+    if status == "pending":
+        return None
+    if status == "rejected":
+        return "rejected"
+    if status == "cancelled":
+        return "cancelled"
+    if status != "fulfilled":
+        return None
+
+    delivery = str(delivery_state or "").strip().lower()
+    if delivery in {"error", "cancelled"}:
+        return delivery
+    return "complete"
+
+
+class ActivityService:
+    """Service for per-user activity dismissals and terminal history snapshots."""
+
+    def __init__(self, db_path: str):
+        self._db_path = db_path
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA foreign_keys = ON")
+        return conn
+
+    @staticmethod
+    def _coerce_positive_int(value: Any, field: str) -> int:
+        try:
+            parsed = int(value)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"{field} must be an integer") from exc
+        if parsed < 1:
+            raise ValueError(f"{field} must be a positive integer")
+        return parsed
+
+    @staticmethod
+    def _row_to_dict(row: sqlite3.Row | None) -> dict[str, Any] | None:
+        return dict(row) if row is not None else None
+
+    @staticmethod
+    def _parse_json_column(value: Any) -> Any:
+        if not isinstance(value, str):
+            return None
+        try:
+            return json.loads(value)
+        except (ValueError, TypeError):
+            return None
+
+    def _build_legacy_request_snapshot(
+        self,
+        conn: sqlite3.Connection,
+        request_id: int,
+    ) -> tuple[dict[str, Any] | None, str | None]:
+        request_row = conn.execute(
+            """
+            SELECT
+                id,
+                user_id,
+                status,
+                delivery_state,
+                request_level,
+                book_data,
+                release_data,
+                note,
+                admin_note,
+                created_at,
+                reviewed_at
+            FROM download_requests
+            WHERE id = ?
+            """,
+            (request_id,),
+        ).fetchone()
+        if request_row is None:
+            return None, None
+
+        row_dict = dict(request_row)
+        book_data = self._parse_json_column(row_dict.get("book_data"))
+        release_data = self._parse_json_column(row_dict.get("release_data"))
+        if not isinstance(book_data, dict):
+            book_data = {}
+        if not isinstance(release_data, dict):
+            release_data = {}
+
+        snapshot = {
+            "kind": "request",
+            "request": {
+                "id": int(row_dict["id"]),
+                "user_id": row_dict.get("user_id"),
+                "status": row_dict.get("status"),
+                "delivery_state": row_dict.get("delivery_state"),
+                "request_level": row_dict.get("request_level"),
+                "book_data": book_data,
+                "release_data": release_data,
+                "note": row_dict.get("note"),
+                "admin_note": row_dict.get("admin_note"),
+                "created_at": row_dict.get("created_at"),
+                "updated_at": row_dict.get("reviewed_at") or row_dict.get("created_at"),
+            },
+        }
+        final_status = _request_final_status(row_dict.get("status"), row_dict.get("delivery_state"))
+        return snapshot, final_status
+
+    def record_terminal_snapshot(
+        self,
+        *,
+        user_id: int | None,
+        item_type: str,
+        item_key: str,
+        origin: str,
+        final_status: str,
+        snapshot: dict[str, Any],
+        request_id: int | None = None,
+        source_id: str | None = None,
+        terminal_at: str | None = None,
+    ) -> dict[str, Any]:
+        """Record a durable terminal-state snapshot for an activity item."""
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_item_key = _normalize_item_key(item_key)
+        normalized_origin = _normalize_origin(origin)
+        normalized_final_status = _normalize_final_status(final_status)
+        if not isinstance(snapshot, dict):
+            raise ValueError("snapshot must be an object")
+
+        if user_id is not None:
+            user_id = self._coerce_positive_int(user_id, "user_id")
+        if request_id is not None:
+            request_id = self._coerce_positive_int(request_id, "request_id")
+        if source_id is not None and not isinstance(source_id, str):
+            raise ValueError("source_id must be a string when provided")
+        if source_id is not None:
+            source_id = source_id.strip() or None
+
+        effective_terminal_at = terminal_at if isinstance(terminal_at, str) and terminal_at.strip() else _now_timestamp()
+        serialized_snapshot = json.dumps(snapshot, separators=(",", ":"), ensure_ascii=False)
+
+        conn = self._connect()
+        try:
+            cursor = conn.execute(
+                """
+                INSERT INTO activity_log (
+                    user_id,
+                    item_type,
+                    item_key,
+                    request_id,
+                    source_id,
+                    origin,
+                    final_status,
+                    snapshot_json,
+                    terminal_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    user_id,
+                    normalized_item_type,
+                    normalized_item_key,
+                    request_id,
+                    source_id,
+                    normalized_origin,
+                    normalized_final_status,
+                    serialized_snapshot,
+                    effective_terminal_at,
+                ),
+            )
+            snapshot_id = int(cursor.lastrowid)
+            conn.commit()
+            row = conn.execute(
+                "SELECT * FROM activity_log WHERE id = ?",
+                (snapshot_id,),
+            ).fetchone()
+            payload = self._row_to_dict(row)
+            if payload is None:
+                raise ValueError("Failed to read back recorded activity snapshot")
+            return payload
+        finally:
+            conn.close()
+
+    def get_latest_activity_log_id(self, *, item_type: str, item_key: str) -> int | None:
+        """Get the newest snapshot ID for an item key."""
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_item_key = _normalize_item_key(item_key)
+        conn = self._connect()
+        try:
+            row = conn.execute(
+                """
+                SELECT id
+                FROM activity_log
+                WHERE item_type = ? AND item_key = ?
+                ORDER BY terminal_at DESC, id DESC
+                LIMIT 1
+                """,
+                (normalized_item_type, normalized_item_key),
+            ).fetchone()
+            if row is None:
+                return None
+            return int(row["id"])
+        finally:
+            conn.close()
+
+    def dismiss_item(
+        self,
+        *,
+        user_id: int,
+        item_type: str,
+        item_key: str,
+        activity_log_id: int | None = None,
+    ) -> dict[str, Any]:
+        """Dismiss an item for a specific user (upsert)."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_item_key = _normalize_item_key(item_key)
+        normalized_log_id = (
+            self._coerce_positive_int(activity_log_id, "activity_log_id")
+            if activity_log_id is not None
+            else self.get_latest_activity_log_id(
+                item_type=normalized_item_type,
+                item_key=normalized_item_key,
+            )
+        )
+
+        conn = self._connect()
+        try:
+            conn.execute(
+                """
+                INSERT INTO activity_dismissals (
+                    user_id,
+                    item_type,
+                    item_key,
+                    activity_log_id,
+                    dismissed_at
+                )
+                VALUES (?, ?, ?, ?, ?)
+                ON CONFLICT(user_id, item_type, item_key)
+                DO UPDATE SET
+                    activity_log_id = excluded.activity_log_id,
+                    dismissed_at = excluded.dismissed_at
+                """,
+                (
+                    normalized_user_id,
+                    normalized_item_type,
+                    normalized_item_key,
+                    normalized_log_id,
+                    _now_timestamp(),
+                ),
+            )
+            conn.commit()
+            row = conn.execute(
+                """
+                SELECT *
+                FROM activity_dismissals
+                WHERE user_id = ? AND item_type = ? AND item_key = ?
+                """,
+                (normalized_user_id, normalized_item_type, normalized_item_key),
+            ).fetchone()
+            payload = self._row_to_dict(row)
+            if payload is None:
+                raise ValueError("Failed to read back dismissal row")
+            return payload
+        finally:
+            conn.close()
+
+    def dismiss_many(self, *, user_id: int, items: Iterable[dict[str, Any]]) -> int:
+        """Dismiss many items for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_items: list[tuple[str, str, int | None]] = []
+        for item in items:
+            if not isinstance(item, dict):
+                raise ValueError("items must contain objects")
+            normalized_item_type = _normalize_item_type(item.get("item_type"))
+            normalized_item_key = _normalize_item_key(item.get("item_key"))
+            raw_log_id = item.get("activity_log_id")
+            normalized_log_id = (
+                self._coerce_positive_int(raw_log_id, "activity_log_id")
+                if raw_log_id is not None
+                else self.get_latest_activity_log_id(
+                    item_type=normalized_item_type,
+                    item_key=normalized_item_key,
+                )
+            )
+            normalized_items.append((normalized_item_type, normalized_item_key, normalized_log_id))
+
+        if not normalized_items:
+            return 0
+
+        conn = self._connect()
+        try:
+            timestamp = _now_timestamp()
+            for item_type, item_key, activity_log_id in normalized_items:
+                conn.execute(
+                    """
+                    INSERT INTO activity_dismissals (
+                        user_id,
+                        item_type,
+                        item_key,
+                        activity_log_id,
+                        dismissed_at
+                    )
+                    VALUES (?, ?, ?, ?, ?)
+                    ON CONFLICT(user_id, item_type, item_key)
+                    DO UPDATE SET
+                        activity_log_id = excluded.activity_log_id,
+                        dismissed_at = excluded.dismissed_at
+                    """,
+                    (
+                        normalized_user_id,
+                        item_type,
+                        item_key,
+                        activity_log_id,
+                        timestamp,
+                    ),
+                )
+            conn.commit()
+            return len(normalized_items)
+        finally:
+            conn.close()
+
+    def get_dismissal_set(self, user_id: int) -> list[dict[str, str]]:
+        """Return dismissed item keys for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT item_type, item_key
+                FROM activity_dismissals
+                WHERE user_id = ?
+                ORDER BY dismissed_at DESC, id DESC
+                """,
+                (normalized_user_id,),
+            ).fetchall()
+            return [
+                {
+                    "item_type": str(row["item_type"]),
+                    "item_key": str(row["item_key"]),
+                }
+                for row in rows
+            ]
+        finally:
+            conn.close()
+
+    def clear_dismissals_for_item_keys(
+        self,
+        *,
+        user_id: int,
+        item_type: str,
+        item_keys: Iterable[str],
+    ) -> int:
+        """Clear dismissals for one user + item type + item keys."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_keys = {
+            _normalize_item_key(item_key)
+            for item_key in item_keys
+            if isinstance(item_key, str) and item_key.strip()
+        }
+        if not normalized_keys:
+            return 0
+
+        conn = self._connect()
+        try:
+            cursor = conn.executemany(
+                """
+                DELETE FROM activity_dismissals
+                WHERE user_id = ? AND item_type = ? AND item_key = ?
+                """,
+                (
+                    (normalized_user_id, normalized_item_type, item_key)
+                    for item_key in normalized_keys
+                ),
+            )
+            conn.commit()
+            return int(cursor.rowcount or 0)
+        finally:
+            conn.close()
+
+    def get_history(self, user_id: int, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
+        """Return paged dismissal history for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_limit = max(1, min(int(limit), 200))
+        normalized_offset = max(0, int(offset))
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT
+                    d.id,
+                    d.user_id,
+                    d.item_type,
+                    d.item_key,
+                    d.activity_log_id,
+                    d.dismissed_at,
+                    l.snapshot_json,
+                    l.origin,
+                    l.final_status,
+                    l.terminal_at,
+                    l.request_id,
+                    l.source_id
+                FROM activity_dismissals d
+                LEFT JOIN activity_log l ON l.id = d.activity_log_id
+                WHERE d.user_id = ?
+                ORDER BY d.dismissed_at DESC, d.id DESC
+                LIMIT ? OFFSET ?
+                """,
+                (normalized_user_id, normalized_limit, normalized_offset),
+            ).fetchall()
+
+            payload: list[dict[str, Any]] = []
+            for row in rows:
+                row_dict = dict(row)
+                raw_snapshot_json = row_dict.pop("snapshot_json", None)
+                snapshot_payload = None
+                if isinstance(raw_snapshot_json, str):
+                    try:
+                        snapshot_payload = json.loads(raw_snapshot_json)
+                    except (ValueError, TypeError):
+                        snapshot_payload = None
+
+                if snapshot_payload is None and row_dict.get("item_type") == "request":
+                    request_id = row_dict.get("request_id")
+                    if request_id is None:
+                        request_id = _parse_request_id_from_item_key(row_dict.get("item_key"))
+                    try:
+                        normalized_request_id = int(request_id) if request_id is not None else None
+                    except (TypeError, ValueError):
+                        normalized_request_id = None
+
+                    if normalized_request_id and normalized_request_id > 0:
+                        fallback_snapshot, fallback_final_status = self._build_legacy_request_snapshot(
+                            conn,
+                            normalized_request_id,
+                        )
+                        if fallback_snapshot is not None:
+                            snapshot_payload = fallback_snapshot
+                            if not row_dict.get("origin"):
+                                row_dict["origin"] = "request"
+                            if not row_dict.get("final_status") and fallback_final_status is not None:
+                                row_dict["final_status"] = fallback_final_status
+
+                row_dict["snapshot"] = snapshot_payload
+                payload.append(row_dict)
+            return payload
+        finally:
+            conn.close()
+
+    def get_undismissed_terminal_downloads(self, user_id: int, *, limit: int = 200) -> list[dict[str, Any]]:
+        """Return latest undismissed terminal download snapshots for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_limit = max(1, min(int(limit), 500))
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT
+                    l.id,
+                    l.user_id,
+                    l.item_type,
+                    l.item_key,
+                    l.request_id,
+                    l.source_id,
+                    l.origin,
+                    l.final_status,
+                    l.snapshot_json,
+                    l.terminal_at
+                FROM activity_log l
+                LEFT JOIN activity_dismissals d
+                    ON d.user_id = ?
+                    AND d.item_type = l.item_type
+                    AND d.item_key = l.item_key
+                WHERE l.user_id = ?
+                    AND l.item_type = 'download'
+                    AND l.final_status IN ('complete', 'error', 'cancelled')
+                    AND d.id IS NULL
+                ORDER BY l.terminal_at DESC, l.id DESC
+                LIMIT ?
+                """,
+                (normalized_user_id, normalized_user_id, normalized_limit * 2),
+            ).fetchall()
+
+            payload: list[dict[str, Any]] = []
+            seen_item_keys: set[str] = set()
+            for row in rows:
+                row_dict = dict(row)
+                item_key = str(row_dict.get("item_key") or "")
+                if not item_key or item_key in seen_item_keys:
+                    continue
+                seen_item_keys.add(item_key)
+
+                raw_snapshot_json = row_dict.pop("snapshot_json", None)
+                snapshot_payload = None
+                if isinstance(raw_snapshot_json, str):
+                    try:
+                        snapshot_payload = json.loads(raw_snapshot_json)
+                    except (ValueError, TypeError):
+                        snapshot_payload = None
+                row_dict["snapshot"] = snapshot_payload
+                payload.append(row_dict)
+                if len(payload) >= normalized_limit:
+                    break
+
+            return payload
+        finally:
+            conn.close()
+
+    def clear_history(self, user_id: int) -> int:
+        """Delete all dismissals for a user and return deleted row count."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        conn = self._connect()
+        try:
+            cursor = conn.execute(
+                "DELETE FROM activity_dismissals WHERE user_id = ?",
+                (normalized_user_id,),
+            )
+            conn.commit()
+            return int(cursor.rowcount or 0)
+        finally:
+            conn.close()

shelfmark/core/admin_routes.py

@@ -0,0 +1,439 @@
+"""Admin user management API routes.
+
+Registers /api/admin/users CRUD endpoints for managing users.
+All endpoints require admin session.
+"""
+
+from functools import wraps
+import os
+import sqlite3
+from typing import Any
+
+from flask import Flask, jsonify, request, session
+from werkzeug.security import generate_password_hash
+
+from shelfmark.config.booklore_settings import (
+    get_booklore_library_options,
+    get_booklore_path_options,
+)
+from shelfmark.config.env import CWA_DB_PATH
+from shelfmark.core.admin_settings_routes import (
+    register_admin_settings_routes,
+    validate_user_settings,
+)
+from shelfmark.core.auth_modes import (
+    AUTH_SOURCE_BUILTIN,
+    AUTH_SOURCE_CWA,
+    AUTH_SOURCE_OIDC,
+    AUTH_SOURCE_PROXY,
+    determine_auth_mode,
+    has_local_password_admin,
+    normalize_auth_source,
+)
+from shelfmark.core.cwa_user_sync import sync_cwa_users_from_rows
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+
+
+def _get_user_edit_capabilities(
+    user: dict[str, Any],
+    security_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Return backend-authored capability flags for the user edit form."""
+    auth_source = normalize_auth_source(
+        user.get("auth_source"),
+        user.get("oidc_subject"),
+    )
+    if security_config is None and auth_source == AUTH_SOURCE_OIDC:
+        security_config = load_config_file("security")
+
+    oidc_use_admin_group = bool((security_config or {}).get("OIDC_USE_ADMIN_GROUP", True))
+    role_managed_by_oidc_group = auth_source == AUTH_SOURCE_OIDC and oidc_use_admin_group
+    can_edit_role = auth_source == AUTH_SOURCE_BUILTIN or (
+        auth_source == AUTH_SOURCE_OIDC and not role_managed_by_oidc_group
+    )
+
+    return {
+        "authSource": auth_source,
+        "canSetPassword": auth_source == AUTH_SOURCE_BUILTIN,
+        "canEditRole": can_edit_role,
+        "canEditEmail": auth_source in {AUTH_SOURCE_BUILTIN, AUTH_SOURCE_PROXY},
+        "canEditDisplayName": auth_source != AUTH_SOURCE_OIDC,
+    }
+
+
+def _get_auth_mode():
+    """Get current auth mode from config."""
+    try:
+        config = load_config_file("security")
+        return determine_auth_mode(
+            config,
+            CWA_DB_PATH,
+            has_local_admin=has_local_password_admin(),
+        )
+    except Exception:
+        return "none"
+
+
+def _require_admin(f):
+    """Decorator to require admin session for admin routes.
+
+    In no-auth mode, everyone has access (is_admin defaults True).
+    In auth-required modes, requires an authenticated session with admin role.
+    """
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        auth_mode = _get_auth_mode()
+        if auth_mode != "none":
+            if "user_id" not in session:
+                return jsonify({"error": "Authentication required"}), 401
+            if not session.get("is_admin", False):
+                return jsonify({"error": "Admin access required"}), 403
+        return f(*args, **kwargs)
+    return decorated
+
+
+def _sanitize_user(user: dict) -> dict:
+    """Remove sensitive fields from user dict before returning to client."""
+    sanitized = dict(user)
+    sanitized.pop("password_hash", None)
+    return sanitized
+
+
+def _oidc_role_management_message(security_config: dict[str, Any]) -> str:
+    admin_group = security_config.get("OIDC_ADMIN_GROUP", "")
+    if admin_group:
+        return (
+            "Admin roles for OIDC users are managed by the "
+            f"'{admin_group}' group in your identity provider"
+        )
+    return (
+        "Disable 'Use Admin Group for Authorization' in security settings "
+        "to manage roles manually"
+    )
+
+
+def _is_user_active(user: dict[str, Any], auth_method: str) -> bool:
+    """Determine whether a user can authenticate in the current auth mode."""
+    source = normalize_auth_source(user.get("auth_source"), user.get("oidc_subject"))
+    if source == AUTH_SOURCE_BUILTIN:
+        return auth_method in (AUTH_SOURCE_BUILTIN, AUTH_SOURCE_OIDC)
+    return source == auth_method
+
+
+def _serialize_user(
+    user: dict[str, Any],
+    auth_method: str,
+    security_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Sanitize and enrich a user payload for API responses."""
+    payload = _sanitize_user(user)
+    payload["auth_source"] = normalize_auth_source(
+        payload.get("auth_source"),
+        payload.get("oidc_subject"),
+    )
+    payload["is_active"] = _is_user_active(payload, auth_method)
+    payload["edit_capabilities"] = _get_user_edit_capabilities(
+        payload,
+        security_config=security_config,
+    )
+    return payload
+
+
+def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]:
+    """Sync all users from the Calibre-Web database into users.db."""
+    if not CWA_DB_PATH or not CWA_DB_PATH.exists():
+        raise FileNotFoundError("Calibre-Web database is not available")
+
+    db_path = os.fspath(CWA_DB_PATH)
+    db_uri = f"file:{db_path}?mode=ro&immutable=1"
+    conn = sqlite3.connect(db_uri, uri=True)
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT name, role, email FROM user")
+        rows = cur.fetchall()
+    finally:
+        conn.close()
+
+    return sync_cwa_users_from_rows(user_db, rows)
+
+
+def register_admin_routes(app: Flask, user_db: UserDB) -> None:
+    """Register admin user management routes on the Flask app."""
+
+    @app.route("/api/admin/users", methods=["GET"])
+    @_require_admin
+    def admin_list_users():
+        """List all users."""
+        users = user_db.list_users()
+        auth_mode = _get_auth_mode()
+        security_config = load_config_file("security")
+        return jsonify([
+            _serialize_user(u, auth_mode, security_config=security_config)
+            for u in users
+        ])
+
+    @app.route("/api/admin/users", methods=["POST"])
+    @_require_admin
+    def admin_create_user():
+        """Create a new user with password authentication."""
+        data = request.get_json() or {}
+        auth_mode = _get_auth_mode()
+
+        username = (data.get("username") or "").strip()
+        password = data.get("password", "")
+        email = (data.get("email") or "").strip() or None
+        display_name = (data.get("display_name") or "").strip() or None
+        role = data.get("role", "user")
+
+        if auth_mode in {AUTH_SOURCE_PROXY, AUTH_SOURCE_CWA}:
+            return jsonify({
+                "error": "Local user creation is disabled in this authentication mode",
+                "message": (
+                    "Users are provisioned by your external authentication source. "
+                    "Switch to builtin or OIDC mode to create local users."
+                ),
+            }), 400
+
+        if not username:
+            return jsonify({"error": "Username is required"}), 400
+        if not password or len(password) < 4:
+            return jsonify({"error": "Password must be at least 4 characters"}), 400
+        if role not in ("admin", "user"):
+            return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
+
+        # First user is always admin
+        if not user_db.list_users():
+            role = "admin"
+
+        # Check if username already exists
+        if user_db.get_user(username=username):
+            return jsonify({"error": "Username already exists"}), 409
+
+        password_hash = generate_password_hash(password)
+        try:
+            user = user_db.create_user(
+                username=username,
+                password_hash=password_hash,
+                email=email,
+                display_name=display_name,
+                auth_source=AUTH_SOURCE_BUILTIN,
+                role=role,
+            )
+        except ValueError:
+            return jsonify({"error": "Username already exists"}), 409
+        logger.info(
+            "Shelfmark user created "
+            f"(source=manual_admin_create, created_by={session.get('user_id', 'unknown')}, "
+            f"username={username}, role={role}, auth_source={AUTH_SOURCE_BUILTIN})"
+        )
+        return jsonify(
+            _serialize_user(
+                user,
+                _get_auth_mode(),
+                security_config=load_config_file("security"),
+            )
+        ), 201
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["GET"])
+    @_require_admin
+    def admin_get_user(user_id):
+        """Get a user by ID with their settings."""
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        result = _serialize_user(
+            user,
+            _get_auth_mode(),
+            security_config=load_config_file("security"),
+        )
+        result["settings"] = user_db.get_user_settings(user_id)
+        return jsonify(result)
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["PUT"])
+    @_require_admin
+    def admin_update_user(user_id):
+        """Update user fields and/or settings."""
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        data = request.get_json() or {}
+        security_config = load_config_file("security")
+        auth_source = normalize_auth_source(
+            user.get("auth_source"),
+            user.get("oidc_subject"),
+        )
+        capabilities = _get_user_edit_capabilities(user, security_config=security_config)
+
+        # Handle optional password update
+        password = data.get("password", "")
+        if password:
+            if not capabilities["canSetPassword"]:
+                return jsonify({
+                    "error": f"Cannot set password for {auth_source.upper()} users",
+                    "message": "Password authentication is only available for local users.",
+                }), 400
+            if len(password) < 4:
+                return jsonify({"error": "Password must be at least 4 characters"}), 400
+            user_db.update_user(user_id, password_hash=generate_password_hash(password))
+
+        # Update user fields
+        user_fields = {}
+        for field in ("role", "email", "display_name"):
+            if field in data:
+                user_fields[field] = data[field]
+
+        if "role" in user_fields and user_fields["role"] not in ("admin", "user"):
+            return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
+
+        role_changed = "role" in user_fields and user_fields["role"] != user.get("role")
+        email_changed = "email" in user_fields and user_fields["email"] != user.get("email")
+        display_name_changed = (
+            "display_name" in user_fields
+            and user_fields["display_name"] != user.get("display_name")
+        )
+
+        if role_changed and not capabilities["canEditRole"]:
+            if auth_source == AUTH_SOURCE_OIDC:
+                return jsonify({
+                    "error": "Cannot change role for OIDC user when group-based authorization is enabled",
+                    "message": _oidc_role_management_message(security_config),
+                }), 400
+
+            return jsonify({
+                "error": f"Cannot change role for {auth_source.upper()} users",
+                "message": "Role is managed by the external authentication source.",
+            }), 400
+
+        if email_changed and not capabilities["canEditEmail"]:
+            if auth_source == AUTH_SOURCE_CWA:
+                return jsonify({
+                    "error": "Cannot change email for CWA users",
+                    "message": "Email is synced from Calibre-Web.",
+                }), 400
+
+            return jsonify({
+                "error": "Cannot change email for OIDC users",
+                "message": "Email is managed by your identity provider.",
+            }), 400
+
+        if display_name_changed and not capabilities["canEditDisplayName"]:
+            return jsonify({
+                "error": "Cannot change display name for OIDC users",
+                "message": "Display name is managed by your identity provider.",
+            }), 400
+
+        # Allow demoting the last admin account.
+        # Auth mode resolution automatically falls back to "none" when no
+        # local password admin remains.
+
+        # Avoid unnecessary writes for no-op field updates.
+        for field in ("role", "email", "display_name"):
+            if field in user_fields and user_fields[field] == user.get(field):
+                user_fields.pop(field)
+
+        if user_fields:
+            user_db.update_user(user_id, **user_fields)
+
+        # Update per-user settings
+        if "settings" in data:
+            if not isinstance(data["settings"], dict):
+                return jsonify({"error": "Settings must be an object"}), 400
+
+            validated_settings, validation_errors = validate_user_settings(data["settings"])
+            if validation_errors:
+                return jsonify({
+                    "error": "Invalid settings payload",
+                    "details": validation_errors,
+                }), 400
+
+            user_db.set_user_settings(user_id, validated_settings)
+            # Ensure runtime reads see updated per-user overrides immediately.
+            try:
+                from shelfmark.core.config import config as app_config
+                app_config.refresh()
+            except Exception:
+                pass
+
+        updated = user_db.get_user(user_id=user_id)
+        result = _serialize_user(
+            updated,
+            _get_auth_mode(),
+            security_config=security_config,
+        )
+        result["settings"] = user_db.get_user_settings(user_id)
+        logger.info(f"Admin updated user {user_id}")
+        return jsonify(result)
+
+    @app.route("/api/admin/users/sync-cwa", methods=["POST"])
+    @_require_admin
+    def admin_sync_cwa_users():
+        """Manually sync users from Calibre-Web into users.db."""
+        auth_mode = _get_auth_mode()
+        if auth_mode != AUTH_SOURCE_CWA:
+            return jsonify({
+                "error": "CWA sync is only available when CWA authentication is enabled",
+            }), 400
+
+        try:
+            summary = _sync_all_cwa_users(user_db)
+        except FileNotFoundError:
+            return jsonify({
+                "error": "Calibre-Web database is not available",
+                "message": "Verify app.db is mounted and readable at /auth/app.db.",
+            }), 503
+        except Exception as exc:
+            logger.error(f"Failed to sync CWA users: {exc}")
+            return jsonify({
+                "error": "Failed to sync users from Calibre-Web",
+            }), 500
+
+        message = (
+            f"Synced {summary['total']} CWA users "
+            f"({summary['created']} created, {summary['updated']} updated, "
+            f"{summary.get('deleted', 0)} deleted)."
+        )
+        logger.info(message)
+        return jsonify({
+            "success": True,
+            "message": message,
+            **summary,
+        })
+
+    register_admin_settings_routes(app, user_db, _require_admin)
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["DELETE"])
+    @_require_admin
+    def admin_delete_user(user_id):
+        """Delete a user."""
+        # Prevent self-deletion
+        if session.get("db_user_id") == user_id:
+            return jsonify({"error": "Cannot delete your own account"}), 400
+
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        auth_mode = _get_auth_mode()
+        auth_source = normalize_auth_source(
+            user.get("auth_source"),
+            user.get("oidc_subject"),
+        )
+        if auth_source == AUTH_SOURCE_CWA and auth_source == auth_mode:
+            return jsonify({
+                "error": f"Cannot delete active {auth_source.upper()} users",
+                "message": f"{auth_source.upper()} users are automatically re-provisioned on login.",
+            }), 400
+
+        # Allow deleting the last local admin account.
+        # Auth mode resolution automatically falls back to "none" when no
+        # local password admin remains.
+
+        user_db.delete_user(user_id)
+        logger.info(f"Admin deleted user {user_id}: {user['username']}")
+        return jsonify({"success": True})

shelfmark/core/admin_settings_routes.py

@@ -0,0 +1,230 @@
+"""Admin settings-introspection routes and settings validation helpers."""
+
+from typing import Any, Callable
+
+from flask import Flask, jsonify, request
+
+from shelfmark.config.notifications_settings import (
+    build_notification_test_result,
+    is_valid_notification_url,
+    normalize_notification_routes,
+)
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_settings_overrides import (
+    build_user_preferences_payload as _build_user_preferences_payload,
+    get_ordered_user_overridable_fields as _get_ordered_user_overridable_fields,
+    get_settings_registry as _get_settings_registry,
+)
+from shelfmark.core.user_db import UserDB
+from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
+
+
+def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
+    settings_registry = _get_settings_registry()
+    field_map = settings_registry.get_settings_field_map()
+    overridable_map = settings_registry.get_user_overridable_fields()
+
+    valid: dict[str, Any] = {}
+    errors: list[str] = []
+    for key, value in settings.items():
+        if key not in field_map:
+            errors.append(f"Unknown setting: {key}")
+        elif key not in overridable_map:
+            errors.append(f"Setting not user-overridable: {key}")
+        else:
+            # null means "clear the per-user override; use global default"
+            if value is None:
+                valid[key] = None
+                continue
+
+            if key in {"REQUEST_POLICY_DEFAULT_EBOOK", "REQUEST_POLICY_DEFAULT_AUDIOBOOK"}:
+                if parse_policy_mode(value) is None:
+                    errors.append(f"Invalid policy mode for {key}: {value}")
+                    continue
+
+            if key == "REQUEST_POLICY_RULES":
+                normalized_rules, rule_errors = validate_policy_rules(value)
+                if rule_errors:
+                    errors.extend(rule_errors)
+                    continue
+                valid[key] = normalized_rules
+                continue
+
+            if key == "USER_NOTIFICATION_ROUTES":
+                normalized_routes = normalize_notification_routes(value)
+                invalid_count = sum(
+                    1
+                    for row in normalized_routes
+                    if row.get("url") and not is_valid_notification_url(str(row.get("url")))
+                )
+                if invalid_count:
+                    errors.append(
+                        (
+                            f"Invalid value for {key}: found {invalid_count} invalid URL(s). "
+                            "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+                        )
+                    )
+                    continue
+                valid[key] = normalized_routes
+                continue
+
+            valid[key] = value
+
+    return valid, errors
+
+
+def build_user_notification_test_response(
+    *,
+    user_id: int,
+    payload: Any,
+) -> tuple[dict[str, Any], int]:
+    from shelfmark.core.config import config as app_config
+
+    routes_input = app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=user_id)
+    if isinstance(payload, dict):
+        if "USER_NOTIFICATION_ROUTES" in payload:
+            routes_input = payload.get("USER_NOTIFICATION_ROUTES")
+        elif "routes" in payload:
+            routes_input = payload.get("routes")
+
+    result = build_notification_test_result(routes_input, scope_label="personal")
+    status_code = 200 if result.get("success", False) else 400
+    return result, status_code
+
+
+def register_admin_settings_routes(
+    app: Flask,
+    user_db: UserDB,
+    require_admin: Callable[[Callable[..., Any]], Callable[..., Any]],
+) -> None:
+    @app.route("/api/admin/download-defaults", methods=["GET"])
+    @require_admin
+    def admin_download_defaults():
+        config = load_config_file("downloads")
+        defaults = {
+            key: ("" if (value := config.get(key, field.default)) is None else value)
+            for key, field in _get_ordered_user_overridable_fields("downloads")
+        }
+
+        security_config = load_config_file("security")
+        defaults["OIDC_ADMIN_GROUP"] = security_config.get("OIDC_ADMIN_GROUP", "")
+        defaults["OIDC_USE_ADMIN_GROUP"] = security_config.get("OIDC_USE_ADMIN_GROUP", True)
+        defaults["OIDC_AUTO_PROVISION"] = security_config.get("OIDC_AUTO_PROVISION", True)
+        return jsonify(defaults)
+
+    @app.route("/api/admin/booklore-options", methods=["GET"])
+    @require_admin
+    def admin_booklore_options():
+        from shelfmark.core import admin_routes
+
+        return jsonify({
+            "libraries": admin_routes.get_booklore_library_options(),
+            "paths": admin_routes.get_booklore_path_options(),
+        })
+
+    @app.route("/api/admin/users/<int:user_id>/delivery-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_delivery_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "downloads")
+        except ValueError:
+            return jsonify({"error": "Downloads settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/notification-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_notification_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "notifications")
+        except ValueError:
+            return jsonify({"error": "Notifications settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/notification-preferences/test", methods=["POST"])
+    @require_admin
+    def admin_test_notification_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        payload = request.get_json(silent=True)
+        result, status_code = build_user_notification_test_response(
+            user_id=user_id,
+            payload=payload,
+        )
+        return jsonify(result), status_code
+
+    @app.route("/api/admin/settings/overrides-summary", methods=["GET"])
+    @require_admin
+    def admin_settings_overrides_summary():
+        settings_registry = _get_settings_registry()
+
+        tab_name = (request.args.get("tab") or "downloads").strip()
+        if not settings_registry.get_settings_tab(tab_name):
+            return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404
+
+        overridable_keys = list(settings_registry.get_user_overridable_fields(tab_name=tab_name))
+        keys_payload: dict[str, dict[str, Any]] = {}
+
+        for user_record in user_db.list_users():
+            user_settings = user_db.get_user_settings(user_record["id"])
+            if not isinstance(user_settings, dict):
+                continue
+
+            for key in overridable_keys:
+                if key not in user_settings or user_settings[key] is None:
+                    continue
+                entry = keys_payload.setdefault(key, {"count": 0, "users": []})
+                entry["users"].append({
+                    "userId": user_record["id"],
+                    "username": user_record["username"],
+                    "value": user_settings[key],
+                })
+
+        for summary in keys_payload.values():
+            summary["count"] = len(summary["users"])
+
+        return jsonify({"tab": tab_name, "keys": keys_payload})
+
+    @app.route("/api/admin/users/<int:user_id>/effective-settings", methods=["GET"])
+    @require_admin
+    def admin_get_effective_settings(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        from shelfmark.core.config import config as app_config
+        from shelfmark.core.settings_registry import is_value_from_env
+
+        field_map = _get_settings_registry().get_user_overridable_fields()
+        user_settings = user_db.get_user_settings(user_id)
+        tab_config_cache: dict[str, dict[str, Any]] = {}
+        effective: dict[str, dict[str, Any]] = {}
+
+        for key, (field, tab_name) in sorted(field_map.items()):
+            value = app_config.get(key, field.default, user_id=user_id)
+            source = "default"
+
+            if field.env_supported and is_value_from_env(field):
+                source = "env_var"
+            elif key in user_settings and user_settings[key] is not None:
+                source = "user_override"
+                value = user_settings[key]
+            else:
+                tab_config = tab_config_cache.setdefault(tab_name, load_config_file(tab_name))
+                if key in tab_config:
+                    source = "global_config"
+
+            effective[key] = {"value": value, "source": source}
+
+        return jsonify(effective)

shelfmark/core/auth_modes.py

@@ -0,0 +1,126 @@
+"""Authentication mode, auth-source normalization, and admin access policy helpers."""
+
+import os
+from typing import Any, Mapping
+
+AUTH_SOURCE_BUILTIN = "builtin"
+AUTH_SOURCE_OIDC = "oidc"
+AUTH_SOURCE_PROXY = "proxy"
+AUTH_SOURCE_CWA = "cwa"
+AUTH_SOURCES = (
+    AUTH_SOURCE_BUILTIN,
+    AUTH_SOURCE_OIDC,
+    AUTH_SOURCE_PROXY,
+    AUTH_SOURCE_CWA,
+)
+AUTH_SOURCE_SET = frozenset(AUTH_SOURCES)
+_ALWAYS_ADMIN_SETTINGS_TABS = frozenset({"security", "users"})
+
+
+def has_local_password_admin(user_db: Any | None = None) -> bool:
+    """Return True when at least one local admin with a password exists."""
+    try:
+        db = user_db
+        if db is None:
+            from shelfmark.core.user_db import UserDB
+
+            config_root = os.environ.get("CONFIG_DIR", "/config")
+            db = UserDB(os.path.join(config_root, "users.db"))
+            db.initialize()
+
+        return any(
+            user.get("password_hash") and user.get("role") == "admin"
+            for user in db.list_users()
+        )
+    except Exception:
+        return False
+
+
+def normalize_auth_source(
+    source: Any,
+    oidc_subject: Any = None,
+) -> str:
+    """Resolve a stable auth source value from persisted fields."""
+    normalized = str(source or "").strip().lower()
+    if normalized in AUTH_SOURCE_SET:
+        return normalized
+    if oidc_subject:
+        return AUTH_SOURCE_OIDC
+    return AUTH_SOURCE_BUILTIN
+
+
+def determine_auth_mode(
+    security_config: Mapping[str, Any],
+    cwa_db_path: Any | None,
+    *,
+    has_local_admin: bool = True,
+) -> str:
+    """Determine active auth mode from security config and runtime prerequisites."""
+    auth_mode = security_config.get("AUTH_METHOD", "none")
+
+    if auth_mode == AUTH_SOURCE_CWA and cwa_db_path:
+        return AUTH_SOURCE_CWA
+
+    if auth_mode == AUTH_SOURCE_BUILTIN and has_local_admin:
+        return AUTH_SOURCE_BUILTIN
+
+    if auth_mode == AUTH_SOURCE_PROXY and security_config.get("PROXY_AUTH_USER_HEADER"):
+        return AUTH_SOURCE_PROXY
+
+    if (
+        auth_mode == AUTH_SOURCE_OIDC
+        and has_local_admin
+        and security_config.get("OIDC_DISCOVERY_URL")
+        and security_config.get("OIDC_CLIENT_ID")
+    ):
+        return AUTH_SOURCE_OIDC
+
+    return "none"
+
+
+def is_settings_or_onboarding_path(path: str) -> bool:
+    """Return True when request path targets protected admin settings routes."""
+    return path.startswith("/api/settings") or path.startswith("/api/onboarding")
+
+
+def get_settings_tab_from_path(path: str) -> str | None:
+    """Extract tab name from /api/settings/<tab>[...] paths."""
+    if not path.startswith("/api/settings/"):
+        return None
+
+    suffix = path[len("/api/settings/"):]
+    if not suffix:
+        return None
+
+    return suffix.split("/", 1)[0] or None
+
+
+def should_restrict_settings_to_admin(
+    _users_config: Mapping[str, Any],
+) -> bool:
+    """Settings/onboarding is always admin-only."""
+    return True
+
+
+def requires_admin_for_settings_access(
+    path: str,
+    users_config: Mapping[str, Any],
+) -> bool:
+    """Return whether this settings/onboarding request requires admin privileges."""
+    tab_name = get_settings_tab_from_path(path)
+    if tab_name in _ALWAYS_ADMIN_SETTINGS_TABS:
+        return True
+
+    return should_restrict_settings_to_admin(users_config)
+
+
+def get_auth_check_admin_status(
+    _auth_mode: str,
+    _users_config: Mapping[str, Any],
+    session_data: Mapping[str, Any],
+) -> bool:
+    """Resolve /api/auth/check `is_admin` as the session's real admin role."""
+    if "user_id" not in session_data:
+        return False
+
+    return bool(session_data.get("is_admin", False))

shelfmark/core/cache.py

@@ -0,0 +1,172 @@
+"""Thread-safe in-memory cache with TTL support."""
+
+import threading
+import time
+from dataclasses import dataclass
+from functools import wraps
+from typing import Any, Callable, Dict, Optional, TypeVar
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+T = TypeVar("T")
+
+
+@dataclass
+class CacheEntry:
+    """A cached value with expiration time."""
+    value: Any
+    expires_at: float
+
+
+class CacheService:
+    """Thread-safe in-memory cache with TTL support."""
+
+    def __init__(self, max_size: int = 1000):
+        """Initialize cache with max_size entries before eviction."""
+        self._cache: Dict[str, CacheEntry] = {}
+        self._lock = threading.Lock()
+        self._max_size = max_size
+
+    def get(self, key: str) -> Optional[Any]:
+        """Get cached value if not expired."""
+        with self._lock:
+            entry = self._cache.get(key)
+            if entry is None:
+                return None
+
+            if time.time() > entry.expires_at:
+                del self._cache[key]
+                return None
+
+            return entry.value
+
+    def set(self, key: str, value: Any, ttl: int) -> None:
+        """Cache value with TTL in seconds."""
+        with self._lock:
+            # Evict oldest entries if at capacity
+            if len(self._cache) >= self._max_size:
+                self._evict_oldest()
+
+            self._cache[key] = CacheEntry(
+                value=value,
+                expires_at=time.time() + ttl
+            )
+
+    def invalidate(self, key: str) -> bool:
+        """Remove specific cache entry. Returns True if found."""
+        with self._lock:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        with self._lock:
+            self._cache.clear()
+
+    def cleanup_expired(self) -> int:
+        """Remove all expired entries. Returns count removed."""
+        with self._lock:
+            now = time.time()
+            expired_keys = [
+                key for key, entry in self._cache.items()
+                if entry.expires_at < now
+            ]
+            for key in expired_keys:
+                del self._cache[key]
+            return len(expired_keys)
+
+    def _evict_oldest(self) -> None:
+        """Evict ~10% of oldest entries. Called with lock held."""
+        if not self._cache:
+            return
+
+        # Remove ~10% of entries, oldest first
+        entries_to_remove = max(1, len(self._cache) // 10)
+        sorted_entries = sorted(
+            self._cache.items(),
+            key=lambda x: x[1].expires_at
+        )
+
+        for key, _ in sorted_entries[:entries_to_remove]:
+            del self._cache[key]
+
+    def stats(self) -> Dict[str, int]:
+        """Get cache statistics (size, max_size)."""
+        with self._lock:
+            return {
+                "size": len(self._cache),
+                "max_size": self._max_size
+            }
+
+
+# Global cache instance for metadata providers
+_metadata_cache = CacheService(max_size=1000)
+
+
+def get_metadata_cache() -> CacheService:
+    """Get the global metadata cache instance."""
+    return _metadata_cache
+
+
+def cache_key(*args, **kwargs) -> str:
+    """Generate cache key from arguments."""
+    parts = [str(arg) for arg in args]
+    parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
+    return ":".join(parts)
+
+
+def cacheable(
+    ttl: Optional[int] = None,
+    ttl_key: Optional[str] = None,
+    ttl_default: int = 300,
+    key_prefix: str = ""
+):
+    """Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            # Check if metadata caching is enabled
+            from shelfmark.core.config import config
+
+            if not config.get("METADATA_CACHE_ENABLED", True):
+                # Caching disabled, execute function directly
+                return func(*args, **kwargs)
+
+            # Determine TTL: static or from config
+            if ttl is not None:
+                effective_ttl = ttl
+            elif ttl_key:
+                effective_ttl = config.get(ttl_key, ttl_default)
+            else:
+                effective_ttl = ttl_default
+
+            # Generate cache key from function name and arguments
+            # Skip 'self' argument if present (first arg of method)
+            cache_args = args[1:] if args and hasattr(args[0], func.__name__) else args
+
+            key = cache_key(
+                key_prefix or func.__name__,
+                *cache_args,
+                **kwargs
+            )
+
+            # Check cache
+            cached = _metadata_cache.get(key)
+            if cached is not None:
+                return cached
+
+            # Execute function and cache result
+            result = func(*args, **kwargs)
+
+            # Only cache non-None results
+            if result is not None:
+                _metadata_cache.set(key, result, effective_ttl)
+
+            return result
+
+        return wrapper
+    return decorator

shelfmark/core/config.py

@@ -0,0 +1,265 @@
+"""Configuration singleton with ENV > config file > default resolution."""
+
+import os
+import sqlite3
+from threading import Lock
+from typing import Any, Dict, Optional
+
+# Import lazily to avoid circular imports
+_registry_module = None
+_env_module = None
+_user_db_module = None
+
+
+def _get_registry():
+    """Lazy import of settings registry to avoid circular imports."""
+    global _registry_module
+    if _registry_module is None:
+        from shelfmark.core import settings_registry
+        _registry_module = settings_registry
+    return _registry_module
+
+
+def _get_env():
+    """Lazy import of env module for fallback values."""
+    global _env_module
+    if _env_module is None:
+        from shelfmark.config import env
+        _env_module = env
+    return _env_module
+
+
+def _get_user_db_module():
+    """Lazy import of user DB module to avoid optional dependency loops."""
+    global _user_db_module
+    if _user_db_module is None:
+        from shelfmark.core.user_db import UserDB
+        _user_db_module = UserDB
+    return _user_db_module
+
+
+class Config:
+    """
+    Dynamic configuration singleton that provides live settings access.
+
+    Settings are resolved with priority: ENV var > config file > default.
+    Values are cached for performance and can be refreshed when settings change.
+    """
+
+    _instance: Optional['Config'] = None
+    _lock = Lock()
+    def __new__(cls) -> 'Config':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+        self._cache: Dict[str, Any] = {}
+        self._field_map: Dict[str, tuple] = {}  # key -> (field, tab_name)
+        self._cache_lock = Lock()
+        self._user_settings_cache: Dict[int, Dict[str, Any]] = {}
+        self._user_settings_cache_lock = Lock()
+        self._user_db = None
+        self._user_db_load_attempted = False
+        self._initialized = True
+        self._loaded = False
+
+    def _ensure_loaded(self) -> None:
+        """Ensure settings are loaded from the registry."""
+        if self._loaded:
+            return
+        with self._cache_lock:
+            if self._loaded:
+                return
+            self._load_settings()
+
+    def _load_settings(self) -> None:
+        """Load all settings from the registry."""
+        # Ensure all settings modules are imported before loading
+        # This handles cases where config is accessed before settings are registered
+        try:
+            import shelfmark.config.settings  # noqa: F401 - main app settings
+            import shelfmark.config.notifications_settings  # noqa: F401 - notifications settings
+            import shelfmark.release_sources  # noqa: F401 - plugin settings
+            import shelfmark.metadata_providers  # noqa: F401 - plugin settings
+        except ImportError:
+            pass
+
+        registry = _get_registry()
+
+        # On first load, sync ENV values to config files
+        # This ensures ENV values persist even if ENV vars are later removed
+        if not hasattr(self, '_env_synced'):
+            registry.sync_env_to_config()
+            self._env_synced = True
+
+        # Build field map from all registered tabs
+        self._field_map.clear()
+        self._cache.clear()
+
+        for tab in registry.get_all_settings_tabs():
+            for field in tab.fields:
+                # Skip action buttons and headings - they don't have values
+                if isinstance(field, (registry.ActionButton, registry.HeadingField)):
+                    continue
+
+                key = field.key
+                self._field_map[key] = (field, tab.name)
+
+                # Load current value
+                value = registry.get_setting_value(field, tab.name)
+                self._cache[key] = value
+
+        self._loaded = True
+
+    def refresh(self) -> None:
+        """
+        Refresh all cached settings from config files.
+
+        Call this after settings are updated via the UI to ensure
+        the config singleton reflects the new values.
+        """
+        with self._cache_lock:
+            self._loaded = False
+            self._load_settings()
+        with self._user_settings_cache_lock:
+            self._user_settings_cache.clear()
+        self._user_db = None
+        self._user_db_load_attempted = False
+
+    def _get_user_db(self):
+        """Get or initialize a UserDB handle if available."""
+        if self._user_db is not None:
+            return self._user_db
+        if self._user_db_load_attempted:
+            return None
+
+        self._user_db_load_attempted = True
+        try:
+            user_db_cls = _get_user_db_module()
+            db_path = os.path.join(os.environ.get("CONFIG_DIR", "/config"), "users.db")
+            user_db = user_db_cls(db_path)
+            user_db.initialize()
+            self._user_db = user_db
+            return self._user_db
+        except Exception:
+            # Multi-user support is optional; fall back to global config when unavailable.
+            return None
+
+    def _get_user_settings(self, user_id: int) -> Dict[str, Any]:
+        """Get cached per-user settings from user DB."""
+        with self._user_settings_cache_lock:
+            if user_id in self._user_settings_cache:
+                return self._user_settings_cache[user_id]
+
+        user_db = self._get_user_db()
+        if user_db is None:
+            return {}
+
+        try:
+            settings = user_db.get_user_settings(user_id)
+        except (sqlite3.OperationalError, OSError, ValueError, TypeError):
+            return {}
+
+        if not isinstance(settings, dict):
+            settings = {}
+
+        with self._user_settings_cache_lock:
+            self._user_settings_cache[user_id] = settings
+        return settings
+
+    def _get_user_override(self, user_id: int, key: str) -> Any:
+        """Get a user override for a specific key."""
+        user_settings = self._get_user_settings(user_id)
+        return user_settings.get(key)
+
+    def get(self, key: str, default: Any = None, user_id: Optional[int] = None) -> Any:
+        """
+        Get a setting value by key.
+
+        Args:
+            key: The setting key (e.g., 'MAX_RETRY')
+            default: Default value if setting not found
+            user_id: Optional DB user ID for per-user setting overrides
+
+        Returns:
+            The setting value, or default if not found
+        """
+        self._ensure_loaded()
+
+        if key in self._field_map:
+            field, _ = self._field_map[key]
+            registry = _get_registry()
+
+            # Deployment-level ENV values always win.
+            if field.env_supported and registry.is_value_from_env(field):
+                return self._cache.get(key, default)
+
+            # User overrides are only available for explicitly overridable fields.
+            if user_id is not None and getattr(field, "user_overridable", False):
+                user_value = self._get_user_override(user_id, key)
+                if user_value is not None:
+                    return user_value
+
+        return self._cache.get(key, default)
+
+    def __getattr__(self, name: str) -> Any:
+        """
+        Allow attribute-style access to settings.
+
+        Example: config.MAX_RETRY instead of config.get('MAX_RETRY')
+        """
+        # Avoid recursion for internal attributes
+        if name.startswith('_'):
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
+
+        self._ensure_loaded()
+
+        if name in self._cache:
+            return self._cache[name]
+
+        # Fallback to env module for settings not in registry
+        # This ensures backward compatibility during migration
+        env = _get_env()
+        if hasattr(env, name):
+            return getattr(env, name)
+
+        raise AttributeError(f"Setting '{name}' not found in config or env")
+
+    def is_from_env(self, key: str) -> bool:
+        """
+        Check if a setting's value comes from an environment variable.
+
+        Args:
+            key: The setting key
+
+        Returns:
+            True if the value is set via ENV var, False otherwise
+        """
+        self._ensure_loaded()
+
+        if key not in self._field_map:
+            return False
+
+        field, _ = self._field_map[key]
+        registry = _get_registry()
+        return registry.is_value_from_env(field)
+
+    def get_all(self) -> Dict[str, Any]:
+        """
+        Get all cached settings as a dictionary.
+
+        Returns:
+            Dict of all setting keys to their current values
+        """
+        self._ensure_loaded()
+        return dict(self._cache)
+
+
+# Global singleton instance
+config = Config()

shelfmark/core/cwa_user_sync.py

@@ -0,0 +1,94 @@
+"""Helpers for provisioning and syncing Calibre-Web users into users.db."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from shelfmark.core.auth_modes import AUTH_SOURCE_CWA, normalize_auth_source
+from shelfmark.core.external_user_linking import upsert_external_user
+from shelfmark.core.user_db import UserDB
+
+_CWA_ALIAS_SUFFIX = "__cwa"
+
+
+def _normalize_email(value: Any) -> str | None:
+    if value is None:
+        return None
+    email = str(value).strip()
+    return email or None
+
+
+def upsert_cwa_user(
+    user_db: UserDB,
+    cwa_username: str,
+    cwa_email: str | None,
+    role: str,
+    context: str | None = None,
+) -> tuple[dict[str, Any], str]:
+    """Create/update a CWA-backed user with collision-safe matching."""
+    normalized_email = _normalize_email(cwa_email)
+    collision_strategy = "alias" if normalized_email else "takeover"
+    user, action = upsert_external_user(
+        user_db,
+        auth_source="cwa",
+        username=cwa_username,
+        email=normalized_email,
+        role=role,
+        allow_email_link=True,
+        collision_strategy=collision_strategy,
+        alias_suffix=_CWA_ALIAS_SUFFIX,
+        context=context,
+    )
+    if user is None:
+        raise RuntimeError("Unexpected CWA user sync result: no user returned")
+    return user, action
+
+
+def sync_cwa_users_from_rows(
+    user_db: UserDB,
+    rows: Iterable[tuple[Any, Any, Any]],
+) -> dict[str, int]:
+    """Sync CWA users from raw `(name, role_flags, email)` rows."""
+    created = 0
+    updated = 0
+    active_cwa_user_ids: set[int] = set()
+    for username, role_flags, email in rows:
+        normalized_username = str(username or "").strip()
+        if not normalized_username:
+            continue
+
+        role = "admin" if (int(role_flags or 0) & 1) == 1 else "user"
+        user, action = upsert_cwa_user(
+            user_db,
+            cwa_username=normalized_username,
+            cwa_email=_normalize_email(email),
+            role=role,
+            context="cwa_manual_sync",
+        )
+        active_cwa_user_ids.add(int(user["id"]))
+        if action == "created":
+            created += 1
+        else:
+            updated += 1
+
+    deleted = 0
+    for existing_user in user_db.list_users():
+        if normalize_auth_source(
+            existing_user.get("auth_source"),
+            existing_user.get("oidc_subject"),
+        ) != AUTH_SOURCE_CWA:
+            continue
+
+        existing_id = int(existing_user.get("id") or 0)
+        if existing_id in active_cwa_user_ids:
+            continue
+
+        user_db.delete_user(existing_id)
+        deleted += 1
+
+    return {
+        "created": created,
+        "updated": updated,
+        "deleted": deleted,
+        "total": created + updated,
+    }

shelfmark/core/external_user_linking.py

@@ -0,0 +1,283 @@
+"""Shared external identity matching and provisioning helpers."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Literal
+
+from shelfmark.core.auth_modes import normalize_auth_source
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.user_db import UserDB
+
+UNSET = object()
+
+CollisionStrategy = Literal["takeover", "suffix", "alias"]
+MatchReason = Literal[
+    "subject_match",
+    "existing_source_username_match",
+    "unique_email_match",
+]
+
+logger = setup_logger(__name__)
+
+
+def _normalize_username(value: Any) -> str:
+    return str(value or "").strip()
+
+
+def _normalize_email(value: Any) -> str | None:
+    if value is None:
+        return None
+    email = str(value).strip()
+    return email or None
+
+
+def _normalize_display_name(value: Any) -> str | None:
+    if value is None:
+        return None
+    name = str(value).strip()
+    return name or None
+
+
+def _email_key(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def _normalize_role(value: Any) -> str:
+    return "admin" if str(value or "").strip().lower() == "admin" else "user"
+
+
+def _get_by_subject(user_db: UserDB, subject_field: str | None, subject: str | None) -> dict[str, Any] | None:
+    if not subject_field or not subject:
+        return None
+    if subject_field == "oidc_subject":
+        return user_db.get_user(oidc_subject=subject)
+    return None
+
+
+def find_unique_user_by_email(user_db: UserDB, email: str | None) -> dict[str, Any] | None:
+    key = _email_key(_normalize_email(email))
+    if not key:
+        return None
+
+    matches = [u for u in user_db.list_users() if _email_key(u.get("email")) == key]
+    return matches[0] if len(matches) == 1 else None
+
+
+def find_external_user_match(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    username: str,
+    email: str | None,
+    subject_field: str | None = None,
+    subject: str | None = None,
+    allow_email_link: bool = False,
+) -> tuple[dict[str, Any] | None, MatchReason | None]:
+    """Find an existing local user that should be linked to an external identity."""
+    normalized_username = _normalize_username(username)
+    normalized_email = _normalize_email(email)
+
+    by_subject = _get_by_subject(user_db, subject_field, subject)
+    if by_subject is not None:
+        return by_subject, "subject_match"
+
+    by_username = user_db.get_user(username=normalized_username)
+    if by_username and normalize_auth_source(
+        by_username.get("auth_source"),
+        by_username.get("oidc_subject"),
+    ) == auth_source:
+        return by_username, "existing_source_username_match"
+
+    if allow_email_link:
+        return find_unique_user_by_email(user_db, normalized_email), "unique_email_match"
+    return None, None
+
+
+def _build_updates(
+    *,
+    auth_source: str,
+    role: str,
+    sync_role: bool,
+    email: str | None | object,
+    display_name: str | None | object,
+    subject_field: str | None,
+    subject: str | None,
+) -> dict[str, Any]:
+    updates: dict[str, Any] = {"auth_source": auth_source}
+    if sync_role:
+        updates["role"] = _normalize_role(role)
+    if email is not UNSET:
+        updates["email"] = _normalize_email(email)
+    if display_name is not UNSET:
+        updates["display_name"] = _normalize_display_name(display_name)
+    if subject_field == "oidc_subject" and subject:
+        updates["oidc_subject"] = subject
+    return updates
+
+
+def _next_suffix_username(user_db: UserDB, base_username: str) -> str:
+    candidate = base_username
+    suffix = 1
+    while user_db.get_user(username=candidate):
+        candidate = f"{base_username}_{suffix}"
+        suffix += 1
+    return candidate
+
+
+def _find_existing_alias_user(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    alias_base: str,
+) -> dict[str, Any] | None:
+    pattern = re.compile(rf"^{re.escape(alias_base)}(?:_\d+)?$")
+    candidates = [
+        user for user in user_db.list_users()
+        if pattern.match(str(user.get("username") or ""))
+        and normalize_auth_source(user.get("auth_source"), user.get("oidc_subject")) == auth_source
+    ]
+    if not candidates:
+        return None
+    return sorted(candidates, key=lambda user: int(user.get("id") or 0))[0]
+
+
+def _resolve_create_username(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    requested_username: str,
+    strategy: CollisionStrategy,
+    alias_suffix: str,
+) -> tuple[str | None, dict[str, Any] | None, str]:
+    existing = user_db.get_user(username=requested_username)
+    if not existing:
+        return requested_username, None, "new_username_available"
+
+    if strategy == "takeover":
+        return None, existing, "username_collision_takeover"
+
+    if strategy == "suffix":
+        return _next_suffix_username(user_db, requested_username), None, "username_collision_suffix"
+
+    alias_base = f"{requested_username}{alias_suffix}"
+    alias_existing = _find_existing_alias_user(
+        user_db,
+        auth_source=auth_source,
+        alias_base=alias_base,
+    )
+    if alias_existing is not None:
+        return None, alias_existing, "reuse_existing_alias"
+    return _next_suffix_username(user_db, alias_base), None, "username_collision_alias"
+
+
+def upsert_external_user(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    username: str,
+    role: str,
+    email: str | None | object = UNSET,
+    display_name: str | None | object = UNSET,
+    subject_field: str | None = None,
+    subject: str | None = None,
+    allow_email_link: bool = False,
+    sync_role: bool = True,
+    allow_create: bool = True,
+    collision_strategy: CollisionStrategy = "takeover",
+    alias_suffix: str | None = None,
+    context: str | None = None,
+) -> tuple[dict[str, Any] | None, str]:
+    """Create/update a user from an external auth identity.
+
+    Returns `(user, action)` where action is one of:
+    - `"updated"`
+    - `"created"`
+    - `"not_found"` (when `allow_create=False` and no link target exists)
+    """
+    normalized_username = _normalize_username(username)
+    if not normalized_username:
+        raise ValueError("External username is required")
+
+    normalized_email = _normalize_email(email) if email is not UNSET else None
+    normalized_display_name = (
+        _normalize_display_name(display_name) if display_name is not UNSET else None
+    )
+    normalized_role = _normalize_role(role)
+
+    matched, match_reason = find_external_user_match(
+        user_db,
+        auth_source=auth_source,
+        username=normalized_username,
+        email=normalized_email,
+        subject_field=subject_field,
+        subject=subject,
+        allow_email_link=allow_email_link,
+    )
+    updates = _build_updates(
+        auth_source=auth_source,
+        role=normalized_role,
+        sync_role=sync_role,
+        email=normalized_email if email is not UNSET else UNSET,
+        display_name=normalized_display_name if display_name is not UNSET else UNSET,
+        subject_field=subject_field,
+        subject=subject,
+    )
+    if matched is not None:
+        user_db.update_user(matched["id"], **updates)
+        mapped = user_db.get_user(user_id=matched["id"]) or matched
+        logger.info(
+            "External user mapped to existing Shelfmark user "
+            f"(source={auth_source}, context={context or 'unspecified'}, reason={match_reason}, "
+            f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
+            f"shelfmark_username={mapped['username']})"
+        )
+        return mapped, "updated"
+
+    if not allow_create:
+        logger.info(
+            "External user could not be mapped and creation is disabled "
+            f"(source={auth_source}, context={context or 'unspecified'}, "
+            f"external_username={normalized_username})"
+        )
+        return None, "not_found"
+
+    resolved_alias_suffix = alias_suffix or f"__{auth_source}"
+    create_username, takeover_target, create_reason = _resolve_create_username(
+        user_db,
+        auth_source=auth_source,
+        requested_username=normalized_username,
+        strategy=collision_strategy,
+        alias_suffix=resolved_alias_suffix,
+    )
+    if takeover_target is not None:
+        user_db.update_user(takeover_target["id"], **updates)
+        mapped = user_db.get_user(user_id=takeover_target["id"]) or takeover_target
+        logger.info(
+            "External user mapped to existing Shelfmark user "
+            f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
+            f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
+            f"shelfmark_username={mapped['username']})"
+        )
+        return mapped, "updated"
+
+    create_kwargs: dict[str, Any] = {
+        "username": create_username,
+        "auth_source": auth_source,
+        "role": normalized_role,
+    }
+    if email is not UNSET:
+        create_kwargs["email"] = normalized_email
+    if display_name is not UNSET:
+        create_kwargs["display_name"] = normalized_display_name
+    if subject_field == "oidc_subject" and subject:
+        create_kwargs["oidc_subject"] = subject
+
+    created = user_db.create_user(**create_kwargs)
+    logger.info(
+        "External user created Shelfmark user "
+        f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
+        f"external_username={normalized_username}, shelfmark_user_id={created['id']}, "
+        f"shelfmark_username={created['username']})"
+    )
+    return created, "created"

shelfmark/core/image_cache.py

@@ -0,0 +1,569 @@
+"""Disk-based image cache with LRU eviction."""
+
+import json
+import os
+import threading
+import time
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Image type detection via magic bytes
+IMAGE_SIGNATURES = {
+    b'\xff\xd8\xff': ('image/jpeg', 'jpg'),
+    b'\x89PNG\r\n\x1a\n': ('image/png', 'png'),
+    b'GIF87a': ('image/gif', 'gif'),
+    b'GIF89a': ('image/gif', 'gif'),
+    b'RIFF': ('image/webp', 'webp'),  # WebP starts with RIFF
+}
+
+# HTTP headers for image fetching
+FETCH_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36',
+    'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+}
+
+# Maximum image size to fetch (5 MB)
+MAX_IMAGE_SIZE = 5 * 1024 * 1024
+
+# Negative cache TTL (for failed fetches) - 1 hour
+NEGATIVE_CACHE_TTL = 3600
+
+# Transient failure cache TTL (for timeouts/connection errors) - 60 seconds
+# Short enough to retry soon, long enough to prevent spam during one page view
+TRANSIENT_CACHE_TTL = 60
+
+
+def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
+    """Detect image type from magic bytes.
+
+    Args:
+        data: Image data bytes
+
+    Returns:
+        Tuple of (content_type, extension) or None if not recognized
+    """
+    for signature, (content_type, ext) in IMAGE_SIGNATURES.items():
+        if data.startswith(signature):
+            return content_type, ext
+
+    # Special case for WebP - check for WEBP after RIFF
+    if data.startswith(b'RIFF') and len(data) > 12 and data[8:12] == b'WEBP':
+        return 'image/webp', 'webp'
+
+    return None
+
+
+class ImageCacheService:
+    """Persistent image cache with LRU eviction and TTL support."""
+
+    def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0):
+        """Initialize the image cache.
+
+        Args:
+            cache_dir: Directory to store cached images
+            max_size_mb: Maximum cache size in megabytes
+            ttl_seconds: Time-to-live in seconds (0 = forever)
+        """
+        self.cache_dir = cache_dir
+        self.max_size_bytes = max_size_mb * 1024 * 1024
+        self.ttl_seconds = ttl_seconds
+        self.index_path = cache_dir / "cache_index.json"
+        self._lock = threading.RLock()
+        self._index: Dict[str, Dict[str, Any]] = {}
+
+        # Stats tracking
+        self._hits = 0
+        self._misses = 0
+
+        # Ensure cache directory exists
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load existing index and sync with files on disk (once at startup)
+        self._load_index()
+        self._sync_index_with_files()
+
+    def _load_index(self) -> None:
+        """Load cache index from disk."""
+        if not self.index_path.exists():
+            self._index = {}
+            return
+
+        try:
+            with open(self.index_path, 'r') as f:
+                self._index = json.load(f)
+        except (json.JSONDecodeError, IOError):
+            self._index = {}
+
+    def _sync_index_with_files(self) -> None:
+        """Sync cache index with actual files on disk.
+
+        - Adds entries for files that exist but aren't in index
+        - Removes entries for files that no longer exist (non-negative only)
+        - Preserves negative cache entries (they have no files)
+        """
+        image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
+        added_count = 0
+        removed_count = 0
+
+        # Build set of files that exist on disk
+        existing_files: Dict[str, Path] = {}
+        for file_path in self.cache_dir.iterdir():
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() not in image_extensions:
+                continue
+            existing_files[file_path.stem] = file_path
+
+        # Add files that aren't in the index
+        for cache_id, file_path in existing_files.items():
+            if cache_id in self._index:
+                continue
+
+            ext = file_path.suffix.lstrip('.')
+            stat = file_path.stat()
+
+            # Detect content type
+            try:
+                with open(file_path, 'rb') as f:
+                    header = f.read(16)
+                detected = _detect_image_type(header)
+                content_type = detected[0] if detected else f'image/{ext}'
+            except IOError:
+                content_type = f'image/{ext}'
+
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': stat.st_size,
+                'cached_at': stat.st_mtime,
+                'accessed_at': stat.st_mtime,
+            }
+            added_count += 1
+
+        # Remove index entries for missing files (skip negative cache entries)
+        stale_entries = []
+        for cache_id, entry in self._index.items():
+            if entry.get('negative', False):
+                continue  # Negative entries don't have files
+            if cache_id not in existing_files:
+                stale_entries.append(cache_id)
+
+        for cache_id in stale_entries:
+            del self._index[cache_id]
+            removed_count += 1
+
+        if added_count > 0 or removed_count > 0:
+            self._save_index()
+
+    def _save_index(self) -> None:
+        """Save cache index to disk."""
+        try:
+            # Write to temp file first, then rename for atomicity
+            temp_path = self.index_path.with_suffix('.tmp')
+            with open(temp_path, 'w') as f:
+                json.dump(self._index, f)
+            temp_path.rename(self.index_path)
+        except IOError:
+            pass
+
+    def _get_image_path(self, cache_id: str, ext: str) -> Path:
+        """Get the file path for a cached image."""
+        return self.cache_dir / f"{cache_id}.{ext}"
+
+    def _is_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a cache entry is expired."""
+        if self.ttl_seconds == 0:
+            return False
+        return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
+
+    def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a negative cache entry is expired.
+
+        Transient failures (timeouts) expire after TRANSIENT_CACHE_TTL (60s).
+        Permanent failures (404s) expire after NEGATIVE_CACHE_TTL (1 hour).
+        """
+        if not entry.get('negative', False):
+            return False
+
+        cached_at = entry.get('cached_at', 0)
+        ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
+        return (time.time() - cached_at) > ttl
+
+    def _calculate_total_size(self) -> int:
+        """Calculate total size of cached images."""
+        return sum(entry.get('size', 0) for entry in self._index.values())
+
+    def _evict_if_needed(self, required_space: int = 0) -> None:
+        """Evict old entries if cache is over size limit.
+
+        Uses LRU eviction based on accessed_at timestamp.
+        """
+        current_size = self._calculate_total_size()
+        target_size = self.max_size_bytes - required_space
+
+        if current_size <= target_size:
+            return
+
+        # Sort entries by accessed_at (oldest first)
+        sorted_entries = sorted(
+            self._index.items(),
+            key=lambda x: x[1].get('accessed_at', 0)
+        )
+
+        evicted_count = 0
+        for cache_id, entry in sorted_entries:
+            if current_size <= target_size:
+                break
+
+            # Delete the image file
+            ext = entry.get('ext', 'jpg')
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                if image_path.exists():
+                    image_path.unlink()
+            except IOError:
+                pass
+
+            # Update tracking
+            current_size -= entry.get('size', 0)
+            del self._index[cache_id]
+            evicted_count += 1
+
+        if evicted_count > 0:
+            self._save_index()
+
+    def get(self, cache_id: str) -> Optional[Tuple[bytes, str]]:
+        """Get a cached image.
+
+        Args:
+            cache_id: Cache key (book ID or composite key)
+
+        Returns:
+            Tuple of (image_data, content_type) or None if not cached/expired
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+
+            # Try reloading from disk if not found (handles multiprocess case)
+            if not entry:
+                self._load_index()
+                entry = self._index.get(cache_id)
+                if not entry:
+                    self._misses += 1
+                    return None
+
+            # Check for negative cache (failed fetch)
+            if entry.get('negative', False):
+                if self._is_negative_expired(entry):
+                    # Negative cache expired, allow retry
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+                # Still in negative cache, return None (don't retry)
+                return None
+
+            # Check for expired entry
+            if self._is_expired(entry):
+                # Remove expired entry
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+                del self._index[cache_id]
+                self._save_index()
+                self._misses += 1
+                return None
+
+            # Try to read the cached image
+            ext = entry.get('ext', 'jpg')
+            content_type = entry.get('content_type', 'image/jpeg')
+            image_path = self._get_image_path(cache_id, ext)
+
+            try:
+                if not image_path.exists():
+                    # File missing, remove from index
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+
+                with open(image_path, 'rb') as f:
+                    data = f.read()
+
+                # Update accessed time
+                entry['accessed_at'] = time.time()
+                self._save_index()
+
+                self._hits += 1
+                return data, content_type
+
+            except IOError:
+                self._misses += 1
+                return None
+
+    def put(self, cache_id: str, data: bytes, content_type: str) -> bool:
+        """Store an image in the cache.
+
+        Args:
+            cache_id: Cache key
+            data: Image data bytes
+            content_type: MIME type of the image
+
+        Returns:
+            True if stored successfully
+        """
+        with self._lock:
+            # Detect image type for extension
+            detected = _detect_image_type(data)
+            if detected:
+                content_type, ext = detected
+            else:
+                # Fall back to content-type header
+                if 'jpeg' in content_type or 'jpg' in content_type:
+                    ext = 'jpg'
+                elif 'png' in content_type:
+                    ext = 'png'
+                elif 'gif' in content_type:
+                    ext = 'gif'
+                elif 'webp' in content_type:
+                    ext = 'webp'
+                else:
+                    ext = 'jpg'  # Default
+
+            image_size = len(data)
+
+            # Evict if needed to make room
+            self._evict_if_needed(image_size)
+
+            # Write image to disk
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                with open(image_path, 'wb') as f:
+                    f.write(data)
+            except IOError:
+                return False
+
+            # Update index
+            now = time.time()
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': image_size,
+                'cached_at': now,
+                'accessed_at': now,
+                'negative': False,
+            }
+            self._save_index()
+            return True
+
+    def put_negative(self, cache_id: str, transient: bool = False) -> None:
+        """Store a negative cache entry (failed fetch).
+
+        Args:
+            cache_id: Cache key
+            transient: If True, uses shorter TTL (for timeouts/connection errors)
+        """
+        with self._lock:
+            self._index[cache_id] = {
+                'negative': True,
+                'transient': transient,
+                'cached_at': time.time(),
+            }
+            self._save_index()
+
+    def delete(self, cache_id: str) -> bool:
+        """Delete a single cache entry.
+
+        Args:
+            cache_id: Cache key
+
+        Returns:
+            True if entry existed and was deleted
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+            if not entry:
+                return False
+
+            # Delete file if it exists
+            if not entry.get('negative', False):
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+
+            del self._index[cache_id]
+            self._save_index()
+            return True
+
+    def clear(self) -> int:
+        """Clear all cached images.
+
+        Returns:
+            Number of entries cleared
+        """
+        with self._lock:
+            count = len(self._index)
+
+            # Delete all image files
+            for cache_id, entry in self._index.items():
+                if not entry.get('negative', False):
+                    ext = entry.get('ext', 'jpg')
+                    image_path = self._get_image_path(cache_id, ext)
+                    try:
+                        if image_path.exists():
+                            image_path.unlink()
+                    except IOError:
+                        pass
+
+            # Clear index
+            self._index = {}
+            self._save_index()
+
+            # Reset stats
+            self._hits = 0
+            self._misses = 0
+
+            return count
+
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dict with size, count, hit rate, etc.
+        """
+        with self._lock:
+            total_size = self._calculate_total_size()
+            entry_count = len(self._index)
+            negative_count = sum(1 for e in self._index.values() if e.get('negative', False))
+            total_requests = self._hits + self._misses
+            hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0
+
+            return {
+                'entry_count': entry_count,
+                'negative_count': negative_count,
+                'total_size_bytes': total_size,
+                'total_size_mb': round(total_size / (1024 * 1024), 2),
+                'max_size_mb': self.max_size_bytes / (1024 * 1024),
+                'hits': self._hits,
+                'misses': self._misses,
+                'hit_rate': round(hit_rate, 1),
+            }
+
+    def fetch_and_cache(self, cache_id: str, url: str) -> Optional[Tuple[bytes, str]]:
+        """Fetch an image from URL and cache it.
+
+        Args:
+            cache_id: Cache key
+            url: URL to fetch from
+
+        Returns:
+            Tuple of (image_data, content_type) or None on failure
+        """
+        try:
+
+            response = requests.get(
+                url,
+                timeout=(5, 10),
+                headers=FETCH_HEADERS,
+                stream=True,
+            )
+            response.raise_for_status()
+
+            # Validate content type
+            content_type = response.headers.get('content-type', '')
+            if not content_type.startswith('image/'):
+                self.put_negative(cache_id)
+                return None
+
+            # Read with size limit
+            data = BytesIO()
+            for chunk in response.iter_content(chunk_size=8192):
+                data.write(chunk)
+                if data.tell() > MAX_IMAGE_SIZE:
+                    self.put_negative(cache_id)
+                    return None
+
+            image_data = data.getvalue()
+
+            if not image_data:
+                self.put_negative(cache_id)
+                return None
+
+            # Store in cache
+            if self.put(cache_id, image_data, content_type):
+                # Get the actual content type from detection
+                detected = _detect_image_type(image_data)
+                if detected:
+                    content_type = detected[0]
+                return image_data, content_type
+
+            return None
+
+        except requests.exceptions.Timeout:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.ConnectionError:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.HTTPError as e:
+            is_404 = e.response is not None and e.response.status_code == 404
+            self.put_negative(cache_id, transient=not is_404)
+            return None
+        except Exception:
+            return None
+
+
+# Singleton instance (initialized lazily when config is available)
+_instance: Optional[ImageCacheService] = None
+_instance_lock = threading.Lock()
+
+
+def get_image_cache() -> ImageCacheService:
+    """Get the singleton image cache instance.
+
+    Lazily initializes using config values.
+    """
+    global _instance
+
+    if _instance is None:
+        with _instance_lock:
+            if _instance is None:
+                from shelfmark.core.config import config
+                from shelfmark.config.env import CONFIG_DIR
+
+                cache_dir = CONFIG_DIR / "covers"
+                max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
+                ttl_days = config.get("COVERS_CACHE_TTL", 0)
+                ttl_seconds = ttl_days * 86400 if ttl_days > 0 else 0
+
+                _instance = ImageCacheService(
+                    cache_dir=cache_dir,
+                    max_size_mb=max_size_mb,
+                    ttl_seconds=ttl_seconds,
+                )
+                logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
+
+    return _instance
+
+
+def reset_image_cache() -> None:
+    """Reset the singleton instance (for testing or config changes)."""
+    global _instance
+    with _instance_lock:
+        _instance = None

shelfmark/core/logger.py

@@ -1,15 +1,17 @@
-"""Centralized logging configuration for the book downloader application."""
+"""Logging configuration and custom logger with error tracing."""
 
 import logging
 import sys
 from pathlib import Path
 from logging.handlers import RotatingFileHandler
-from env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
 from typing import Any
 
+from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
+
+
 class CustomLogger(logging.Logger):
     """Custom logger class with additional error_trace method."""
-    
+
     def error_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log an error message with full stack trace."""
         self.log_resource_usage()
@@ -21,56 +23,74 @@ class CustomLogger(logging.Logger):
         self.log_resource_usage()
         kwargs.pop('exc_info', None)
         self.warning(msg, *args, exc_info=True, **kwargs)
-    
+
     def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log an info message with full stack trace."""
-        self.log_resource_usage()
+        """Log an info message (stack trace only if exception active)."""
         kwargs.pop('exc_info', None)
-        self.info(msg, *args, exc_info=True, **kwargs)
-    
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.info(msg, *args, exc_info=has_exception, **kwargs)
+
     def debug_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log a debug message with full stack trace."""
-        self.log_resource_usage()
+        """Log a debug message (stack trace only if exception active)."""
         kwargs.pop('exc_info', None)
-        self.debug(msg, *args, exc_info=True, **kwargs)
-    
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.debug(msg, *args, exc_info=has_exception, **kwargs)
+
     def log_resource_usage(self):
-        import psutil
-        memory = psutil.virtual_memory()
-        available_mb = memory.available / (1024 * 1024)
-        memory_used_mb = memory.used / (1024 * 1024)
-        cpu_percent = psutil.cpu_percent()
-        self.debug(f"Container Memory: Available={available_mb:.2f} MB, Used={memory_used_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
+        # Best-effort only; this should never raise during exception logging.
+        try:
+            import psutil
+
+            # Sum RSS of all processes for actual app memory (container-friendly),
+            # but fall back gracefully on platforms that restrict process enumeration.
+            app_memory_mb = 0.0
+            try:
+                for proc in psutil.process_iter(['memory_info']):
+                    try:
+                        mem = proc.info.get('memory_info')
+                        if mem:
+                            app_memory_mb += mem.rss / (1024 * 1024)
+                    except (psutil.NoSuchProcess, psutil.AccessDenied, KeyError, AttributeError):
+                        continue
+            except (PermissionError, psutil.AccessDenied, OSError):
+                try:
+                    app_memory_mb = psutil.Process().memory_info().rss / (1024 * 1024)
+                except Exception:
+                    app_memory_mb = 0.0
+
+            memory = psutil.virtual_memory()
+            system_used_mb = memory.used / (1024 * 1024)
+            available_mb = memory.available / (1024 * 1024)
+            cpu_percent = psutil.cpu_percent()
+            self.debug(
+                f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, "
+                f"Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%"
+            )
+        except Exception:
+            # Avoid breaking the original log call if psutil is missing or restricted.
+            return
 
 
 def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     """Set up and configure a logger instance.
-    
+
     Args:
         name: The name of the logger instance
         log_file: Optional path to log file. If None, logs only to stdout/stderr
-        
+
     Returns:
         CustomLogger: Configured logger instance with error_trace method
     """
     # Register our custom logger class
     logging.setLoggerClass(CustomLogger)
-    
+
     # Create logger as CustomLogger instance
     logger = CustomLogger(name)
-    log_level = logging.INFO
-    if LOG_LEVEL == "DEBUG":
-        log_level = logging.DEBUG
-    elif LOG_LEVEL == "INFO":
-        log_level = logging.INFO
-    elif LOG_LEVEL == "WARNING":
-        log_level = logging.WARNING
-    elif LOG_LEVEL == "ERROR":
-        log_level = logging.ERROR
-    elif LOG_LEVEL == "CRITICAL":
-        log_level = logging.CRITICAL
+    log_level = getattr(logging, LOG_LEVEL, logging.INFO)
     logger.setLevel(log_level)
-    
+
     formatter = logging.Formatter(
         '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
     )
@@ -81,13 +101,13 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     console_handler.setLevel(log_level)
     console_handler.addFilter(lambda record: record.levelno < logging.ERROR)  # Only allow logs below ERROR to stdout
     logger.addHandler(console_handler)
-    
+
     # Error handler for stderr
     error_handler = logging.StreamHandler(sys.stderr)
     error_handler.setLevel(logging.ERROR) # Error and above go to stderr
     error_handler.setFormatter(formatter)
     logger.addHandler(error_handler)
-    
+
     # File handler if log file is specified
     try:
         if ENABLE_LOGGING:
@@ -105,4 +125,3 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
         logger.error_trace(f"Failed to create log file: {e}", exc_info=True)
 
     return logger
-

shelfmark/core/mirrors.py

@@ -0,0 +1,254 @@
+"""Centralized mirror configuration for all download sources."""
+
+from typing import List
+
+from shelfmark.core.utils import normalize_http_url
+
+# Lazy import to avoid circular imports
+_config_module = None
+
+
+def _get_config():
+    """Lazy import of config module to avoid circular imports."""
+    global _config_module
+    if _config_module is None:
+        from shelfmark.core.config import config
+        _config_module = config
+    return _config_module
+
+
+# Default mirror lists (hardcoded fallbacks)
+DEFAULT_AA_MIRRORS = [
+    "https://annas-archive.gl",
+    "https://annas-archive.li",
+]
+
+DEFAULT_LIBGEN_MIRRORS = [
+    "https://libgen.gl",
+    "https://libgen.li",
+    "https://libgen.bz",
+    "https://libgen.la",
+    "https://libgen.vg",
+]
+
+DEFAULT_ZLIB_MIRRORS = [
+    "https://z-lib.fm",
+    "https://z-lib.gs",
+    "https://z-lib.id",
+    "https://z-library.sk",
+    "https://zlibrary-global.se",
+]
+
+DEFAULT_WELIB_MIRRORS = [
+    "https://welib.org",
+]
+
+
+def _normalize_mirror_url(url: str) -> str:
+    return normalize_http_url(url, default_scheme="https")
+
+
+def get_aa_mirrors() -> List[str]:
+    """
+    Get Anna's Archive mirrors.
+
+    Returns:
+        Ordered list of AA mirror URLs.
+
+        If AA_MIRROR_URLS is configured, it is treated as the full list.
+        Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended.
+
+        Notes:
+        - The list is used to populate the AA mirror dropdown in Settings.
+        - When AA_BASE_URL is set to 'auto', mirrors are tried in the order listed.
+    """
+    config = _get_config()
+
+    mirrors: list[str] = []
+
+    configured_list = config.get("AA_MIRROR_URLS", None)
+    if isinstance(configured_list, list):
+        for url in configured_list:
+            normalized = _normalize_mirror_url(str(url))
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+    elif isinstance(configured_list, str) and configured_list.strip():
+        # Allow comma-separated env/manual configs.
+        for url in configured_list.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    if not mirrors:
+        mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
+        mirrors = [url for url in mirrors if url]
+
+        # Backwards-compatible append-only behavior for legacy configs/env.
+        additional = config.get("AA_ADDITIONAL_URLS", "")
+        if additional:
+            for url in additional.split(","):
+                normalized = _normalize_mirror_url(url)
+                if normalized and normalized not in mirrors:
+                    mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_libgen_mirrors() -> List[str]:
+    """
+    Get LibGen mirrors: defaults + any additional from config.
+
+    Returns:
+        List of LibGen mirror URLs (defaults first, then custom additions).
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_mirrors() -> List[str]:
+    """
+    Get Z-Library mirrors, with primary first.
+
+    Returns:
+        List of Z-Library mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_primary_url() -> str:
+    """
+    Get the primary Z-Library mirror URL.
+
+    Returns:
+        Primary Z-Library mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+
+
+def get_zlib_url_template() -> str:
+    """
+    Get Z-Library URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_zlib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_welib_mirrors() -> List[str]:
+    """
+    Get Welib mirrors, with primary first.
+
+    Returns:
+        List of Welib mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_WELIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("WELIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_welib_primary_url() -> str:
+    """
+    Get the primary Welib mirror URL.
+
+    Returns:
+        Primary Welib mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+
+
+def get_welib_url_template() -> str:
+    """
+    Get Welib URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_welib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_zlib_cookie_domains() -> set:
+    """
+    Get set of Z-Library domains that need full cookie handling.
+
+    Used by internal_bypasser for CF bypass cookie management.
+
+    Returns:
+        Set of domain strings (without protocol).
+    """
+    domains = set()
+
+    # Add all default domains
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized:
+            domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+            domains.add(domain)
+
+    # Add custom domains
+    config = _get_config()
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized:
+                domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+                domains.add(domain)
+
+    return domains

shelfmark/core/models.py

@@ -0,0 +1,182 @@
+"""Data structures and models used across the application."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from enum import Enum
+import re
+import time
+
+
+def build_filename(
+    title: str,
+    author: Optional[str] = None,
+    year: Optional[str] = None,
+    fmt: Optional[str] = None,
+) -> str:
+    parts = []
+    if author:
+        parts.append(author)
+        parts.append(" - ")
+    parts.append(title)
+    if year:
+        parts.append(f" ({year})")
+
+    filename = "".join(parts)
+    filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
+
+    if fmt:
+        filename = f"{filename}.{fmt}"
+
+    return filename
+
+
+class QueueStatus(str, Enum):
+    """Enum for possible book queue statuses."""
+    QUEUED = "queued"
+    RESOLVING = "resolving"
+    LOCATING = "locating"
+    DOWNLOADING = "downloading"
+    COMPLETE = "complete"
+    AVAILABLE = "available"
+    ERROR = "error"
+    DONE = "done"
+    CANCELLED = "cancelled"
+
+
+class SearchMode(str, Enum):
+    DIRECT = "direct"
+    UNIVERSAL = "universal"
+
+
+@dataclass
+class QueueItem:
+    """Queue item with priority and metadata."""
+    book_id: str
+    priority: int
+    added_time: float
+
+    def __lt__(self, other):
+        """Compare items for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+
+@dataclass
+class DownloadTask:
+    task_id: str                                # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
+    source: str                                 # Handler name ("direct_download", "prowlarr")
+    title: str                                  # Display title for queue sidebar
+
+    # Display info for queue sidebar
+    author: Optional[str] = None
+    year: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    preview: Optional[str] = None
+    content_type: Optional[str] = None  # "book (fiction)", "audiobook", "magazine", etc.
+    source_url: Optional[str] = None  # Original release URL used by source-specific handlers
+
+    # Series info (for library naming templates)
+    series_name: Optional[str] = None
+    series_position: Optional[float] = None  # Float for novellas (e.g., 1.5)
+    subtitle: Optional[str] = None  # Book subtitle for naming templates
+
+    # Hardlinking support
+    original_download_path: Optional[str] = None  # Path in download client (for hardlinking)
+
+    # Search mode - determines post-download processing behavior
+    # See SearchMode enum for behavioral differences
+    search_mode: Optional[SearchMode] = None
+
+    # Output selection for post-processing.
+    # This is captured at queue time so in-flight tasks are not affected if the user changes settings later.
+    output_mode: Optional[str] = None  # e.g. "folder", "booklore", "email"
+    output_args: Dict[str, Any] = field(default_factory=dict)  # Per-output parameters (e.g. email recipient)
+
+    # User association (multi-user support)
+    user_id: Optional[int] = None  # DB user ID who queued this download
+    username: Optional[str] = None  # Username for {User} template variable
+    request_id: Optional[int] = None  # Origin request ID when queued from request fulfilment
+
+    # Runtime state
+    priority: int = 0
+    added_time: float = field(default_factory=time.time)
+    progress: float = 0.0
+    status: QueueStatus = QueueStatus.QUEUED
+    status_message: Optional[str] = None
+    download_path: Optional[str] = None
+
+    def __lt__(self, other):
+        """Compare tasks for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+    def get_filename(self) -> str:
+        """Build sanitized filename from task metadata."""
+        if self.download_path:
+            return Path(self.download_path).name
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class BookInfo:
+    """Data class representing book information."""
+    id: str
+    title: str
+    preview: Optional[str] = None
+    author: Optional[str] = None
+    publisher: Optional[str] = None
+    year: Optional[str] = None
+    language: Optional[str] = None
+    content: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    info: Optional[Dict[str, List[str]]] = None
+    description: Optional[str] = None
+    download_urls: List[str] = field(default_factory=list)
+    download_path: Optional[str] = None
+    priority: int = 0
+    progress: Optional[float] = None
+    status_message: Optional[str] = None  # Detailed status message for UI display
+    added_time: Optional[float] = None  # Timestamp when added to queue
+    source: str = "direct_download"  # Release source handler to use for downloads
+    source_url: Optional[str] = None  # Link to source page (e.g., Anna's Archive)
+
+    def get_filename(self, fallback_url: Optional[str] = None) -> str:
+        """Build sanitized filename: 'Author - Title (Year).format'
+
+        Resolves format from self.format, download_urls, or fallback_url.
+
+        Args:
+            fallback_url: URL to extract format from if not already known
+
+        Returns:
+            Sanitized filename safe for filesystem use
+        """
+        # Resolve format if needed
+        if not self.format:
+            urls = [self.download_urls[0]] if self.download_urls else []
+            if fallback_url:
+                urls.append(fallback_url)
+            for url in urls:
+                ext = url.split(".")[-1].lower()
+                if ext and len(ext) <= 5 and ext.isalnum():
+                    self.format = ext
+                    break
+
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class SearchFilters:
+    """Filters for book search queries."""
+    isbn: Optional[List[str]] = None
+    author: Optional[List[str]] = None
+    title: Optional[List[str]] = None
+    lang: Optional[List[str]] = None
+    sort: Optional[str] = None
+    content: Optional[List[str]] = None
+    format: Optional[List[str]] = None

shelfmark/core/naming.py

@@ -0,0 +1,248 @@
+"""Template-based naming for library organization."""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional, Union, Mapping
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+# Known variable tokens, sorted longest-first to avoid partial matches
+# e.g., "SeriesPosition" must match before "Series"
+KNOWN_TOKENS = [
+    'seriesposition',
+    'partnumber',
+    'subtitle',
+    'author',
+    'series',
+    'title',
+    'year',
+    'user',
+]
+
+# Match any {...} block for template parsing
+BRACE_PATTERN = re.compile(r'\{([^}]+)\}')
+
+# Characters that are invalid in filenames on various filesystems
+INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def _sanitize(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for filesystem use."""
+    if not name:
+        return ""
+
+    sanitized = INVALID_CHARS.sub('_', name)
+    sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized)  # Strip whitespace and dots
+    sanitized = re.sub(r'_+', '_', sanitized)  # Collapse underscores
+    return sanitized[:max_length]
+
+
+def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for use as a filename or path component."""
+    return _sanitize(name, max_length)
+
+
+# Alias for backwards compatibility
+sanitize_path_component = sanitize_filename
+
+
+def format_series_position(position: Optional[Union[str, int, float]]) -> str:
+    if position is None:
+        return ""
+
+    # Display as integer if whole number
+    if isinstance(position, float) and position.is_integer():
+        return str(int(position))
+
+    return str(position)
+
+
+# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
+PAD_NUMBERS_PATTERN = re.compile(r'\d+')
+
+
+def natural_sort_key(path: Union[str, Path]) -> str:
+    """Generate a sort key with padded numbers for natural sorting."""
+    filename = Path(path).name.lower()
+    return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
+
+
+def assign_part_numbers(
+    files: list[Path],
+    zero_pad_width: int = 2,
+) -> list[tuple[Path, str]]:
+    """Sort files naturally and assign sequential part numbers (1, 2, 3...)."""
+    if not files:
+        return []
+
+    sorted_files = sorted(files, key=natural_sort_key)
+    return [
+        (file_path, str(part_num).zfill(zero_pad_width))
+        for part_num, file_path in enumerate(sorted_files, start=1)
+    ]
+
+
+def parse_naming_template(
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    *,
+    allow_path_separators: bool = True,
+) -> str:
+    if not template:
+        return ""
+
+    # Normalize metadata keys to lowercase for case-insensitive matching
+    normalized = {k.lower(): v for k, v in metadata.items()}
+
+    def find_token(content: str) -> tuple[Optional[str], int]:
+        content_lower = content.lower()
+        for token in KNOWN_TOKENS:
+            idx = content_lower.find(token)
+            if idx != -1:
+                return token, idx
+        return None, -1
+
+    def token_value(token: str) -> str:
+        value = normalized.get(token)
+        if token == 'seriesposition':
+            value = format_series_position(value)
+        if value is None:
+            return ""
+        return str(value).strip()
+
+    def render_block(content: str) -> Optional[str]:
+        token, idx = find_token(content)
+        if token is None:
+            return None
+
+        prefix = content[:idx]
+        suffix = content[idx + len(token):]
+        value = token_value(token)
+        if not value:
+            return ""
+
+        if not allow_path_separators:
+            value = value.replace("/", "_")
+        value = sanitize_filename(value)
+        return f"{prefix}{value}{suffix}"
+
+    # Process brace blocks in order so we can support conditional literal blocks like:
+    # { - Part }{PartNumber}
+    matches = list(BRACE_PATTERN.finditer(template))
+    if not matches:
+        result = template
+    else:
+        parts: list[str] = []
+        cursor = 0
+        for idx, match in enumerate(matches):
+            parts.append(template[cursor:match.start()])
+            content = match.group(1)
+            rendered = render_block(content)
+
+            if rendered is not None:
+                parts.append(rendered)
+            else:
+                conditional_literal = False
+                include_literal = False
+                if idx + 1 < len(matches) and match.end() == matches[idx + 1].start():
+                    next_content = matches[idx + 1].group(1)
+                    next_token, _next_idx = find_token(next_content)
+                    if next_token is not None:
+                        conditional_literal = True
+                        include_literal = bool(token_value(next_token))
+                if include_literal:
+                    parts.append(content)
+                elif not conditional_literal:
+                    # Preserve blocks that look like literal text, but treat bare unknown
+                    # placeholders as missing variables.
+                    if re.search(r"\s", content):
+                        parts.append(match.group(0))
+
+            cursor = match.end()
+
+        parts.append(template[cursor:])
+        result = "".join(parts)
+
+    # Clean up any double slashes that might result from empty tokens
+    result = re.sub(r'/+', '/', result)
+
+    # Remove leading/trailing slashes
+    result = result.strip('/')
+
+    # Clean up any orphaned separators (e.g., " - " at start/end, or " -  - ")
+    result = re.sub(r'^[\s\-_.]+', '', result)
+    result = re.sub(r'[\s\-_.]+$', '', result)
+    result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
+
+    # Clean up empty parentheses/brackets
+    result = re.sub(r'\(\s*\)', '', result)
+    result = re.sub(r'\[\s*\]', '', result)
+
+    # Final trim of any trailing separators left after cleanup
+    result = re.sub(r'[\s\-_.]+$', '', result)
+
+    return result
+
+
+def build_library_path(
+    base_path: str,
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    extension: Optional[str] = None,
+) -> Path:
+    relative = parse_naming_template(template, metadata, allow_path_separators=True)
+
+    if not relative:
+        # Fallback to title if template produces empty result
+        title = metadata.get('Title') or metadata.get('title') or 'Unknown'
+        relative = sanitize_filename(str(title))
+
+    # Remove any path traversal attempts
+    relative = relative.replace('..', '')
+
+    base = Path(base_path).resolve()
+    full_path = (base / relative).resolve()
+
+    # Verify the path is within the base directory
+    try:
+        full_path.relative_to(base)
+    except ValueError:
+        raise ValueError(f"Path traversal detected: template would escape library directory")
+
+    if extension:
+        ext = extension.lstrip('.')
+        # Don't use with_suffix() - it replaces everything after the first dot
+        # e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
+        full_path = Path(f"{full_path}.{ext}")
+
+    return full_path
+
+
+def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
+    """Check if two paths are on the same filesystem."""
+    path1 = Path(path1)
+    path2 = Path(path2)
+
+    def get_device(p: Path) -> Optional[int]:
+        try:
+            while not p.exists():
+                p = p.parent
+                if p == p.parent:
+                    break
+            return os.stat(p).st_dev
+        except (OSError, PermissionError) as e:
+            logger.debug(f"Cannot stat {p}: {e}")
+            return None
+
+    dev1 = get_device(path1)
+    dev2 = get_device(path2)
+
+    if dev1 is None or dev2 is None:
+        logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
+        return False
+
+    return dev1 == dev2

shelfmark/core/notifications.py

@@ -0,0 +1,380 @@
+"""Apprise notification dispatch for global and per-user events."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Iterable
+
+try:
+    import apprise
+except Exception:  # pragma: no cover - exercised in tests via monkeypatch
+    apprise = None  # type: ignore[assignment]
+
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Small pool for non-blocking dispatch. Notification sends are I/O bound and infrequent.
+_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="Notify")
+_ROUTE_EVENT_ALL = "all"
+_APPRISE_APP_ID = "Shelfmark"
+_APPRISE_APP_DESC = "Shelfmark notifications"
+_APPRISE_LOGO_URL = (
+    "https://raw.githubusercontent.com/calibrain/shelfmark/main/src/frontend/public/logo.png"
+)
+
+
+class NotificationEvent(str, Enum):
+    """Global notification event identifiers."""
+
+    REQUEST_CREATED = "request_created"
+    REQUEST_FULFILLED = "request_fulfilled"
+    REQUEST_REJECTED = "request_rejected"
+    DOWNLOAD_COMPLETE = "download_complete"
+    DOWNLOAD_FAILED = "download_failed"
+
+
+@dataclass
+class NotificationContext:
+    """Context used to render notification templates."""
+
+    event: NotificationEvent
+    title: str
+    author: str
+    username: str | None = None
+    content_type: str | None = None
+    format: str | None = None
+    source: str | None = None
+    admin_note: str | None = None
+    error_message: str | None = None
+
+
+def _normalize_urls(value: Any) -> list[str]:
+    if value is None:
+        return []
+
+    raw_values: list[Any]
+    if isinstance(value, list):
+        raw_values = value
+    elif isinstance(value, str):
+        # Support legacy/manual configs.
+        raw_values = [segment for part in value.splitlines() for segment in part.split(",")]
+    else:
+        raw_values = [value]
+
+    normalized: list[str] = []
+    seen: set[str] = set()
+    for raw_url in raw_values:
+        url = str(raw_url or "").strip()
+        if not url:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        normalized.append(url)
+    return normalized
+
+
+def _normalize_routes(value: Any) -> list[dict[str, str]]:
+    if not isinstance(value, list):
+        return []
+
+    allowed_events = {_ROUTE_EVENT_ALL, *(event.value for event in NotificationEvent)}
+    normalized: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        raw_events = row.get("event")
+        if isinstance(raw_events, list):
+            event_values = raw_events
+        elif isinstance(raw_events, (tuple, set)):
+            event_values = list(raw_events)
+        else:
+            event_values = [raw_events]
+
+        url = str(row.get("url") or "").strip()
+        if not url:
+            continue
+
+        row_events: list[str] = []
+        for raw_event in event_values:
+            event = str(raw_event or "").strip().lower()
+            if event not in allowed_events:
+                continue
+            if event in row_events:
+                continue
+            row_events.append(event)
+
+        if _ROUTE_EVENT_ALL in row_events:
+            row_events = [_ROUTE_EVENT_ALL]
+
+        for event in row_events:
+            key = (event, url)
+            if key in seen:
+                continue
+            seen.add(key)
+
+            normalized.append({"event": event, "url": url})
+
+    return normalized
+
+
+def _resolve_admin_routes() -> list[dict[str, str]]:
+    return _normalize_routes(app_config.get("ADMIN_NOTIFICATION_ROUTES", []))
+
+
+def _normalize_user_id(value: Any) -> int | None:
+    try:
+        user_id = int(value)
+    except (TypeError, ValueError):
+        return None
+    if user_id < 1:
+        return None
+    return user_id
+
+
+def _resolve_user_routes(user_id: int | None) -> list[dict[str, str]]:
+    normalized_user_id = _normalize_user_id(user_id)
+    if normalized_user_id is None:
+        return []
+
+    return _normalize_routes(
+        app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=normalized_user_id)
+    )
+
+
+def _resolve_route_urls_for_event(
+    routes: list[dict[str, str]],
+    event: NotificationEvent,
+) -> list[str]:
+    selected: list[str] = []
+    seen: set[str] = set()
+    event_value = event.value
+
+    for row in routes:
+        row_event = row.get("event", "")
+        if row_event not in {_ROUTE_EVENT_ALL, event_value}:
+            continue
+
+        url = row.get("url", "")
+        if not url or url in seen:
+            continue
+
+        seen.add(url)
+        selected.append(url)
+
+    return selected
+
+
+def _resolve_notify_type(event: NotificationEvent) -> Any:
+    if apprise is None:
+        fallback = {
+            NotificationEvent.REQUEST_CREATED: "info",
+            NotificationEvent.REQUEST_FULFILLED: "success",
+            NotificationEvent.REQUEST_REJECTED: "warning",
+            NotificationEvent.DOWNLOAD_COMPLETE: "success",
+            NotificationEvent.DOWNLOAD_FAILED: "failure",
+        }
+        return fallback[event]
+
+    mapping = {
+        NotificationEvent.REQUEST_CREATED: apprise.NotifyType.INFO,
+        NotificationEvent.REQUEST_FULFILLED: apprise.NotifyType.SUCCESS,
+        NotificationEvent.REQUEST_REJECTED: apprise.NotifyType.WARNING,
+        NotificationEvent.DOWNLOAD_COMPLETE: apprise.NotifyType.SUCCESS,
+        NotificationEvent.DOWNLOAD_FAILED: apprise.NotifyType.FAILURE,
+    }
+    return mapping[event]
+
+
+def _clean_text(value: Any, fallback: str) -> str:
+    text = str(value or "").strip()
+    return text or fallback
+
+
+def _render_message(context: NotificationContext) -> tuple[str, str]:
+    event = context.event
+    title = _clean_text(context.title, "Unknown title")
+    author = _clean_text(context.author, "Unknown author")
+    username = _clean_text(context.username, "A user")
+
+    if event == NotificationEvent.REQUEST_CREATED:
+        return "New Request", f'{username} requested "{title}" by {author}'
+    if event == NotificationEvent.REQUEST_FULFILLED:
+        return "Request Approved", f'Request for "{title}" by {author} was approved.'
+    if event == NotificationEvent.REQUEST_REJECTED:
+        note = _clean_text(context.admin_note, "")
+        note_line = f"\nNote: {note}" if note else ""
+        return "Request Rejected", f'Request for "{title}" by {author} was rejected.{note_line}'
+    if event == NotificationEvent.DOWNLOAD_COMPLETE:
+        return "Download Complete", f'"{title}" by {author} downloaded successfully.'
+
+    error_message = _clean_text(context.error_message, "")
+    error_line = f"\nError: {error_message}" if error_message else ""
+    return "Download Failed", f'Failed to download "{title}" by {author}.{error_line}'
+
+
+def _dispatch_to_apprise(
+    urls: Iterable[str],
+    *,
+    title: str,
+    body: str,
+    notify_type: Any,
+) -> dict[str, Any]:
+    normalized_urls = _normalize_urls(list(urls))
+    if not normalized_urls:
+        return {"success": False, "message": "No notification URLs configured"}
+
+    if apprise is None:
+        return {"success": False, "message": "Apprise is not installed"}
+
+    apobj = _create_apprise_client()
+    if apobj is None:
+        return {"success": False, "message": "Apprise is not installed"}
+    valid_urls = 0
+    invalid_urls = 0
+    for url in normalized_urls:
+        try:
+            added = bool(apobj.add(url))
+        except Exception:
+            added = False
+        if added:
+            valid_urls += 1
+        else:
+            invalid_urls += 1
+
+    if valid_urls == 0:
+        return {
+            "success": False,
+            "message": "No valid notification URLs configured",
+        }
+
+    try:
+        delivered = bool(apobj.notify(title=title, body=body, notify_type=notify_type))
+    except Exception as exc:
+        return {"success": False, "message": f"Notification send failed: {type(exc).__name__}: {exc}"}
+
+    if not delivered:
+        return {"success": False, "message": "Notification delivery failed"}
+
+    message = f"Notification sent to {valid_urls} URL(s)"
+    if invalid_urls:
+        message += f" ({invalid_urls} invalid URL(s) skipped)"
+    return {"success": True, "message": message}
+
+
+def _create_apprise_client() -> Any:
+    if apprise is None:
+        return None
+
+    apprise_cls = getattr(apprise, "Apprise", None)
+    if apprise_cls is None:
+        return None
+
+    apprise_asset_cls = getattr(apprise, "AppriseAsset", None)
+    if apprise_asset_cls is None:
+        return apprise_cls()
+
+    try:
+        asset = apprise_asset_cls(
+            app_id=_APPRISE_APP_ID,
+            app_desc=_APPRISE_APP_DESC,
+            image_url_logo=_APPRISE_LOGO_URL,
+        )
+    except TypeError:
+        # Support older Apprise versions that do not expose image_url_logo.
+        asset = apprise_asset_cls(
+            app_id=_APPRISE_APP_ID,
+            app_desc=_APPRISE_APP_DESC,
+        )
+    except Exception:
+        return apprise_cls()
+
+    try:
+        return apprise_cls(asset=asset)
+    except Exception:
+        return apprise_cls()
+
+
+def _send_admin_event(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> dict[str, Any]:
+    title, body = _render_message(context)
+    notify_type = _resolve_notify_type(event)
+    return _dispatch_to_apprise(urls, title=title, body=body, notify_type=notify_type)
+
+
+def notify_admin(event: NotificationEvent, context: NotificationContext) -> None:
+    """Send a global admin notification for an event if subscribed."""
+    routes = _resolve_admin_routes()
+    urls = _resolve_route_urls_for_event(routes, event)
+    if not urls:
+        return
+
+    try:
+        _executor.submit(_dispatch_admin_async, event, context, urls)
+    except Exception as exc:
+        logger.warning("Failed to queue admin notification '%s': %s", event.value, exc)
+
+
+def notify_user(user_id: int | None, event: NotificationEvent, context: NotificationContext) -> None:
+    """Send a per-user notification for an event if subscribed."""
+    normalized_user_id = _normalize_user_id(user_id)
+    if normalized_user_id is None:
+        return
+
+    routes = _resolve_user_routes(normalized_user_id)
+    urls = _resolve_route_urls_for_event(routes, event)
+    if not urls:
+        return
+
+    try:
+        _executor.submit(_dispatch_user_async, normalized_user_id, event, context, urls)
+    except Exception as exc:
+        logger.warning(
+            "Failed to queue user notification '%s' for user_id=%s: %s",
+            event.value,
+            normalized_user_id,
+            exc,
+        )
+
+
+def _dispatch_admin_async(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> None:
+    result = _send_admin_event(event, context, urls)
+    if not result.get("success", False):
+        logger.warning("Admin notification failed for event '%s': %s", event.value, result.get("message"))
+
+
+def _dispatch_user_async(
+    user_id: int,
+    event: NotificationEvent,
+    context: NotificationContext,
+    urls: list[str],
+) -> None:
+    result = _send_admin_event(event, context, urls)
+    if not result.get("success", False):
+        logger.warning(
+            "User notification failed for event '%s' (user_id=%s): %s",
+            event.value,
+            user_id,
+            result.get("message"),
+        )
+
+
+def send_test_notification(urls: list[str]) -> dict[str, Any]:
+    """Send a synchronous test notification to the provided URLs."""
+    normalized_urls = _normalize_urls(urls)
+    if not normalized_urls:
+        return {"success": False, "message": "No notification URLs configured"}
+
+    test_context = NotificationContext(
+        event=NotificationEvent.REQUEST_CREATED,
+        title="Shelfmark Test Notification",
+        author="Shelfmark",
+        username="Shelfmark",
+    )
+    return _send_admin_event(NotificationEvent.REQUEST_CREATED, test_context, normalized_urls)

shelfmark/core/oidc_auth.py

@@ -0,0 +1,80 @@
+"""OIDC authentication helpers.
+
+Handles group claim parsing, user info extraction, and user provisioning.
+Flask route handlers are registered separately in main.py.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.external_user_linking import upsert_external_user
+from shelfmark.core.user_db import UserDB
+
+def parse_group_claims(id_token: Dict[str, Any], group_claim: str) -> List[str]:
+    """Extract group list from an ID token claim.
+
+    Supports list, comma-separated string, or pipe-separated string.
+    Returns empty list if claim is missing.
+    """
+    raw = id_token.get(group_claim)
+    if raw is None:
+        return []
+    if isinstance(raw, list):
+        return [str(g).strip() for g in raw if str(g).strip()]
+    if isinstance(raw, str):
+        delimiter = "," if "," in raw else "|"
+        return [g.strip() for g in raw.split(delimiter) if g.strip()]
+    return []
+
+
+def extract_user_info(id_token: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract user info from OIDC ID token claims.
+
+    Returns a dict with keys: oidc_subject, username, email, display_name.
+    Falls back through preferred_username -> email -> sub for username.
+    """
+    sub = id_token.get("sub", "")
+    email = id_token.get("email")
+    display_name = id_token.get("name")
+    username = id_token.get("preferred_username") or email or sub
+
+    return {
+        "oidc_subject": sub,
+        "username": username,
+        "email": email,
+        "display_name": display_name,
+    }
+
+
+def provision_oidc_user(
+    db: UserDB,
+    user_info: Dict[str, Any],
+    is_admin: Optional[bool] = None,
+    allow_email_link: bool = False,
+    allow_create: bool = True,
+) -> Optional[Dict[str, Any]]:
+    """Create or update a user from OIDC claims.
+
+    Matching and collision handling use the shared external user linker:
+    - OIDC subject first
+    - optionally unique email linking (when `allow_email_link=True`)
+    - username conflict resolution via numeric suffix.
+
+    Returns None when no existing user is matchable and `allow_create=False`.
+    """
+    oidc_subject = user_info["oidc_subject"]
+    user, _ = upsert_external_user(
+        db,
+        auth_source="oidc",
+        username=user_info["username"] or oidc_subject,
+        role="admin" if is_admin else "user",
+        email=user_info.get("email"),
+        display_name=user_info.get("display_name"),
+        subject_field="oidc_subject",
+        subject=oidc_subject,
+        allow_email_link=allow_email_link,
+        sync_role=is_admin is not None,
+        allow_create=allow_create,
+        collision_strategy="suffix",
+        context="oidc_login",
+    )
+    return user

shelfmark/core/oidc_routes.py

@@ -0,0 +1,210 @@
+"""OIDC Flask route handlers using Authlib.
+
+Registers /api/auth/oidc/login and /api/auth/oidc/callback endpoints.
+Business logic remains in oidc_auth.py.
+"""
+
+from typing import Any
+
+from authlib.jose.errors import InvalidClaimError
+from authlib.integrations.flask_client import OAuth
+from flask import Flask, jsonify, redirect, request, session
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.oidc_auth import (
+    extract_user_info,
+    parse_group_claims,
+    provision_oidc_user,
+)
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+oauth = OAuth()
+
+
+def _normalize_claims(raw_claims: Any) -> dict[str, Any]:
+    """Return a plain dict for claims from Authlib token/userinfo payloads."""
+    if raw_claims is None:
+        return {}
+    if isinstance(raw_claims, dict):
+        return raw_claims
+    if hasattr(raw_claims, "to_dict"):
+        return raw_claims.to_dict()  # type: ignore[no-any-return]
+    try:
+        return dict(raw_claims)
+    except Exception:
+        return {}
+
+
+def _is_email_verified(claims: dict[str, Any]) -> bool:
+    """Normalize provider-specific email_verified values into a strict boolean."""
+    value = claims.get("email_verified", False)
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() == "true"
+    return False
+
+
+def _get_oidc_client() -> tuple[Any, dict[str, Any]]:
+    """Register and return an OIDC client from the current security config."""
+    config = load_config_file("security")
+    discovery_url = config.get("OIDC_DISCOVERY_URL", "")
+    client_id = config.get("OIDC_CLIENT_ID", "")
+
+    if not discovery_url or not client_id:
+        raise ValueError("OIDC not configured")
+
+    configured_scopes = config.get("OIDC_SCOPES", ["openid", "email", "profile"])
+    if isinstance(configured_scopes, list):
+        scope_values = [str(scope).strip() for scope in configured_scopes if str(scope).strip()]
+    elif isinstance(configured_scopes, str):
+        delimiter = "," if "," in configured_scopes else " "
+        scope_values = [scope.strip() for scope in configured_scopes.split(delimiter) if scope.strip()]
+    else:
+        scope_values = []
+
+    scopes = list(dict.fromkeys(["openid"] + scope_values))
+
+    admin_group = config.get("OIDC_ADMIN_GROUP", "")
+    group_claim = config.get("OIDC_GROUP_CLAIM", "groups")
+    use_admin_group = config.get("OIDC_USE_ADMIN_GROUP", True)
+    if admin_group and use_admin_group and group_claim and group_claim not in scopes:
+        scopes.append(group_claim)
+
+    oauth._clients.pop("shelfmark_idp", None)
+    oauth.register(
+        name="shelfmark_idp",
+        client_id=client_id,
+        client_secret=config.get("OIDC_CLIENT_SECRET", ""),
+        server_metadata_url=discovery_url,
+        client_kwargs={
+            "scope": " ".join(scopes),
+            "code_challenge_method": "S256",
+        },
+        overwrite=True,
+    )
+
+    client = oauth.create_client("shelfmark_idp")
+    if client is None:
+        raise RuntimeError("OIDC client initialization failed")
+
+    return client, config
+
+
+def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
+    """Register OIDC authentication routes on the Flask app."""
+    oauth.init_app(app)
+
+    @app.route("/api/auth/oidc/login", methods=["GET"])
+    def oidc_login():
+        """Initiate OIDC login flow and redirect to the provider."""
+        try:
+            client, _ = _get_oidc_client()
+            redirect_uri = request.url_root.rstrip("/") + "/api/auth/oidc/callback"
+            return client.authorize_redirect(redirect_uri)
+        except ValueError:
+            return jsonify({"error": "OIDC not configured"}), 500
+        except Exception as e:
+            logger.error(f"OIDC login error: {e}")
+            return jsonify({"error": "OIDC login failed"}), 500
+
+    @app.route("/api/auth/oidc/callback", methods=["GET"])
+    def oidc_callback():
+        """Handle OIDC callback from identity provider."""
+        try:
+            error = request.args.get("error")
+            if error:
+                logger.warning(f"OIDC callback error from IdP: {error}")
+                return jsonify({"error": "Authentication failed"}), 400
+
+            client, config = _get_oidc_client()
+            try:
+                token = client.authorize_access_token()
+            except InvalidClaimError as e:
+                claim_name = getattr(e, "claim_name", "unknown")
+                discovery_url = str(config.get("OIDC_DISCOVERY_URL", ""))
+                provider_issuer = ""
+                try:
+                    metadata = client.load_server_metadata()
+                    if isinstance(metadata, dict):
+                        provider_issuer = str(metadata.get("issuer", ""))
+                except Exception as metadata_error:
+                    logger.debug(f"OIDC metadata lookup failed during claim diagnostics: {metadata_error}")
+
+                logger.error(
+                    "OIDC callback claim validation failed: claim=%s error=%s discovery_url=%s provider_issuer=%s",
+                    claim_name,
+                    e,
+                    discovery_url or "<unset>",
+                    provider_issuer or "<unknown>",
+                )
+                if claim_name == "iss":
+                    return (
+                        jsonify(
+                            {
+                                "error": (
+                                    "OIDC issuer validation failed. Verify your discovery URL and IdP issuer/"
+                                    "external URL configuration."
+                                )
+                            }
+                        ),
+                        400,
+                    )
+
+                return jsonify({"error": f"OIDC token claim validation failed: {claim_name}"}), 400
+            claims = _normalize_claims(token.get("userinfo"))
+
+            # If userinfo isn't present in token payload, request it explicitly.
+            if not claims:
+                try:
+                    claims = _normalize_claims(client.userinfo(token=token))
+                except TypeError:
+                    claims = _normalize_claims(client.userinfo())
+                except Exception as e:
+                    logger.error(f"Failed to fetch OIDC userinfo: {e}")
+
+            if not claims:
+                raise ValueError("OIDC authentication failed: missing user claims")
+
+            group_claim = config.get("OIDC_GROUP_CLAIM", "groups")
+            admin_group = config.get("OIDC_ADMIN_GROUP", "")
+            use_admin_group = config.get("OIDC_USE_ADMIN_GROUP", True)
+            auto_provision = config.get("OIDC_AUTO_PROVISION", True)
+
+            user_info = extract_user_info(claims)
+            groups = parse_group_claims(claims, group_claim)
+
+            is_admin = None
+            if admin_group and use_admin_group:
+                is_admin = admin_group in groups
+
+            allow_email_link = bool(user_info.get("email")) and _is_email_verified(claims)
+            user = provision_oidc_user(
+                user_db,
+                user_info,
+                is_admin=is_admin,
+                allow_email_link=allow_email_link,
+                allow_create=bool(auto_provision),
+            )
+            if user is None:
+                logger.warning(
+                    f"OIDC login rejected: auto-provision disabled for {user_info['username']}"
+                )
+                return jsonify({"error": "Account not found. Contact your administrator."}), 403
+
+            session["user_id"] = user["username"]
+            session["is_admin"] = user.get("role") == "admin"
+            session["db_user_id"] = user["id"]
+            session.permanent = True
+
+            logger.info(f"OIDC login successful: {user['username']} (admin={is_admin})")
+            return redirect(request.script_root or "/")
+
+        except ValueError as e:
+            logger.error(f"OIDC callback error: {e}")
+            return jsonify({"error": str(e)}), 400
+        except Exception as e:
+            logger.error(f"OIDC callback error: {e}")
+            return jsonify({"error": "Authentication failed"}), 500