Fix theming bug (#722)

- Adds full interaction with Hardcover lists, including adding and
removing from lists + want to read status
- List selection exposed in search results, details modal and release
modal
- Added automatic list dropdown when selecting "list" search
- Added auto-removal of books from a list when downloading from that
specific list page
- Changed search selector to hover-activated

.dockerignore

@@ -37,3 +37,13 @@ dist/
 venv/
 .venv/
 env/
+
+# Frontend build artifacts (built in separate stage)
+src/frontend/node_modules/
+src/frontend/dist/
+src/frontend/.vite/
+
+# Old frontend code (replaced by src/frontend)
+templates/
+static/css/
+static/js/

.github/workflows/build-and-publish-docker-image.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository_owner }}/shelfmark
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -20,12 +20,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - suffix: ""
-            target: cwa-bd
-            image_name_suffix: ""
-          - suffix: "-tor"
-            target: cwa-bd-tor
-            image_name_suffix: "-tor"
+          - target: shelfmark
+          - target: shelfmark-lite
+            image_name_suffix: "-lite"
     steps:
       - name: Get current date
         id: date
@@ -67,6 +64,7 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           build-args: |
             BUILD_VERSION=${{ steps.date.outputs.date }}-${{ github.sha }}
+            RELEASE_VERSION=${{ github.ref_name }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
       
@@ -76,4 +74,71 @@ jobs:
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
+
+  # Create aliases for backwards compatibility
+  create-aliases:
+    needs: build-and-push-images
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    permissions:
+      contents: read
+      packages: write
+    env:
+      # Legacy name for backwards compatibility (hardcoded so it works after rename)
+      LEGACY_NAME: calibre-web-automated-book-downloader
+    steps:
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create legacy aliases
+        run: |
+          # Current image names (follows repo name)
+          STANDARD="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          LITE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-lite"
+
+          # Legacy image names (hardcoded for backwards compatibility)
+          LEGACY="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.LEGACY_NAME }}"
+          LEGACY_TOR="${LEGACY}-tor"
+          LEGACY_EXTBP="${LEGACY}-extbp"
+
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+
+          # Helper function to create alias with all standard tags
+          create_alias() {
+            local SOURCE=$1
+            local ALIAS=$2
+
+            # Always create SHA tag
+            docker buildx imagetools create -t "${ALIAS}:sha-${SHA_SHORT}" "${SOURCE}:sha-${SHA_SHORT}"
+
+            if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+              VERSION="${{ github.ref_name }}"
+              VERSION_NUM="${VERSION#v}"
+              MINOR="${VERSION_NUM%.*}"
+
+              docker buildx imagetools create -t "${ALIAS}:latest" "${SOURCE}:latest"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION_NUM}" "${SOURCE}:${VERSION_NUM}"
+              docker buildx imagetools create -t "${ALIAS}:${MINOR}" "${SOURCE}:${MINOR}"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION}" "${SOURCE}:${VERSION}"
+            else
+              docker buildx imagetools create -t "${ALIAS}:dev" "${SOURCE}:dev"
+            fi
+          }
+
+          # Create legacy aliases pointing to current images
+          # calibre-web-automated-book-downloader → standard image
+          create_alias "${STANDARD}" "${LEGACY}"
+
+          # calibre-web-automated-book-downloader-tor → standard image
+          create_alias "${STANDARD}" "${LEGACY_TOR}"
+
+          # calibre-web-automated-book-downloader-extbp → lite image
+          create_alias "${LITE}" "${LEGACY_EXTBP}"

.gitignore

@@ -227,3 +227,10 @@ pyrightconfig.json
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python
 /downloaded_files
+/.local/
+*.local.*
+AGENTS.md
+.claude/
+.playwright-mcp/
+frontend-dist/
+node_modules/

.vscode/launch.json

@@ -2,20 +2,21 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Current CWABD File",
+      "name": "Python Debugger: Current Shelfmark File",
       "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false,
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader",
-        "TEMP_DIR": "/tmp/cwa-book-downloader",
+        "INGEST_DIR": "/tmp/shelfmark",
+        "TEMP_DIR": "/tmp/shelfmark",
         "LOG_LEVEL": "DEBUG",
-        "LOG_ROOT": "/tmp/cwa-book-downloader",
+        "LOG_ROOT": "/tmp/shelfmark",
         "ENABLE_LOGGING": "true",
         "DOCKERMODE": "false",
-        "DEBUG": "true"
+        "DEBUG": "true",
+        "CUSTOM_DNS": "google",
       },
     },
     {
@@ -26,7 +27,7 @@
       "preLaunchTask": "docker-compose up (dev)", // Spin up dev containers
       "postDebugTask": "docker-compose down (dev)", // Optional: tear them down
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -37,7 +38,7 @@
       "preLaunchTask": "docker-compose up (prod)",
       "postDebugTask": "docker-compose down (prod)",
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -53,9 +54,9 @@
   ],
   "compounds": [
     {
-      "name": "Launch CWA-BD",
+      "name": "Launch Shelfmark",
       "configurations": [
-        "Launch cwa-bd app.py",
+        "Launch Shelfmark app.py",
         "Launch Browser"
       ]
     }

Dockerfile

@@ -1,9 +1,37 @@
+ARG TARGETPLATFORM
+ARG TARGETARCH
+ARG BUILDPLATFORM
+ARG BUILDARCH
+
+# Frontend build stage.
+FROM --platform=$BUILDPLATFORM node:20-alpine AS frontend-builder
+
+# Helpful debug output to see what platforms BuildKit thinks it's using
+RUN echo "BUILDPLATFORM=$BUILDPLATFORM BUILDARCH=$BUILDARCH TARGETPLATFORM=$TARGETPLATFORM TARGETARCH=$TARGETARCH"
+
+WORKDIR /frontend
+
+# Copy frontend package files
+COPY src/frontend/package*.json ./
+
+# Install dependencies (cache mount for faster rebuilds)
+RUN --mount=type=cache,target=/root/.npm \
+    npm ci
+
+# Copy frontend source
+COPY src/frontend/ ./
+
+# Build the frontend
+RUN npm run build
+
 # Use python-slim as the base image
 FROM python:3.10-slim AS base
 
 # Add build argument for version
 ARG BUILD_VERSION
 ENV BUILD_VERSION=${BUILD_VERSION}
+ARG RELEASE_VERSION
+ENV RELEASE_VERSION=${RELEASE_VERSION}
 
 # Set shell to bash with pipefail option
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -17,13 +45,12 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    NAME=Calibre-Web-Automated-Book-Downloader \
+    NAME=Shelfmark \
     PYTHONPATH=/app \
-    # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
+    # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
-    LC_ALL=en_US.UTF-8 \
-    APP_ENV=prod
+    LC_ALL=en_US.UTF-8
 
 # Set ARG for build-time expansion (FLASK_PORT), ENV for runtime access
 ENV FLASK_PORT=8084
@@ -38,18 +65,17 @@ RUN apt-get update && \
     curl \
     # For entrypoint
     dumb-init \
-    # For dumb display
-    xvfb \
-    # For screen recording
-    ffmpeg \
     # For debug
     zip iputils-ping \
     # For user switching
-    sudo \
-    # --- Chromium Browser ---
-    chromium-driver \
-    # For tkinter (pyautogui)
-    python3-tk && \
+    gosu \
+    # --- Tor support (activated via USING_TOR=true) ---
+    tor \
+    supervisor \
+    iptables && \
+    # Configure iptables alternatives for tor.sh compatibility
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
     # Cleanup APT cache *after* all installs in this layer
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
@@ -66,61 +92,72 @@ RUN apt-get update && \
 WORKDIR /app
 
 # Install Python dependencies using pip
-# Upgrade pip first, then copy requirements and install
-# Copying requirements.txt separately leverages build cache
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
-
-# Add this line to grant read/execute permissions to others
-RUN chmod -R o+rx /usr/bin/chromium && \
-    chmod -R o+rx /usr/bin/chromedriver && \
-    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+# Copying requirements files separately leverages build cache
+# Cache mount persists pip cache between builds for faster installs
+COPY requirements-base.txt requirements-shelfmark.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-base.txt
 
 # Copy application code *after* dependencies are installed
 COPY . .
 
+# Copy built frontend from frontend-builder stage
+COPY --from=frontend-builder /frontend/dist /app/frontend-dist
+
 # Final setup: permissions and directories in one layer
 # Only creating directories and setting executable bits.
 # Ownership will be handled by the entrypoint script.
-RUN mkdir -p /var/log/cwa-book-downloader /cwa-book-ingest && \
+RUN mkdir -p /var/log/shelfmark /books && \
     chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
 
 # Expose the application port
 EXPOSE ${FLASK_PORT}
 
 # Add healthcheck for container status
-# This will run as root initially, but check localhost which should work if the app binds correctly.
+# Uses /api/health which doesn't require authentication
 HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
-    CMD curl -s http://localhost:${FLASK_PORT}/request/api/status > /dev/null || exit 1
+    CMD curl -s http://localhost:${FLASK_PORT}/api/health > /dev/null || exit 1
 
 # Use dumb-init as the entrypoint to handle signals properly
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 
-FROM base AS cwa-bd
+FROM base AS shelfmark
 
-# Default command to run the application entrypoint script
-CMD ["/app/entrypoint.sh"]
-
-FROM base AS cwa-bd-tor
-
-ENV USING_TOR=true
-
-# Install Tor and dependencies
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    # --- Tor ---
-    tor \
-    # --- iptables ---
-    iptables && \
-    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
-    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
-    # Cleanup APT cache *after* all installs in this layer
+    # For dumb display
+    xvfb \
+    # For screen recording
+    ffmpeg \
+    # --- Chromium (unpinned - uses latest from Debian repos) ---
+    # Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
+    # This flag is set in internal_bypasser.py _get_browser_args()
+    chromium \
+    chromium-common \
+    # For tkinter (pyautogui)
+    python3-tk \
+    # For RAR extraction
+    unrar-free && \
+    # Create symlink so rarfile library can find unrar
+    ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
+    # Cleanup APT cache
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Override the default command to run Tor
+# Install additional dependencies (requirements file already copied in base stage)
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-shelfmark.txt
+
+# Grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium
+
+# Default command to run the application entrypoint script
+CMD ["/app/entrypoint.sh"]
+
+FROM base AS shelfmark-lite
+
+ENV USING_EXTERNAL_BYPASSER=true
+
 CMD ["/app/entrypoint.sh"]

Makefile

@@ -0,0 +1,90 @@
+.PHONY: help install dev build preview typecheck frontend-test clean up down docker-build refresh restart
+
+# Frontend directory
+FRONTEND_DIR := src/frontend
+
+# Docker compose file
+COMPOSE_FILE := docker-compose.dev.yml
+
+# Default target
+help:
+	@echo "Available targets:"
+	@echo ""
+	@echo "Frontend:"
+	@echo "  install    - Install frontend dependencies"
+	@echo "  dev        - Start development server"
+	@echo "  build      - Build frontend for production"
+	@echo "  preview    - Preview production build"
+	@echo "  typecheck  - Run TypeScript type checking"
+	@echo "  frontend-test - Run frontend unit tests"
+	@echo "  clean      - Remove node_modules and build artifacts"
+	@echo ""
+	@echo "Backend (Docker):"
+	@echo "  up         - Start backend services"
+	@echo "  down       - Stop backend services"
+	@echo "  restart    - Restart backend services (no rebuild)"
+	@echo "  docker-build - Build Docker image"
+	@echo "  refresh    - Rebuild and restart backend services"
+
+# Install dependencies
+install:
+	@echo "Installing frontend dependencies..."
+	cd $(FRONTEND_DIR) && npm install
+
+# Start development server
+dev:
+	@echo "Starting development server..."
+	cd $(FRONTEND_DIR) && npm run dev
+
+# Build for production
+build:
+	@echo "Building frontend for production..."
+	cd $(FRONTEND_DIR) && npm run build
+
+# Preview production build
+preview:
+	@echo "Previewing production build..."
+	cd $(FRONTEND_DIR) && npm run preview
+
+# Type checking
+typecheck:
+	@echo "Running TypeScript type checking..."
+	cd $(FRONTEND_DIR) && npm run typecheck
+
+# Run frontend unit tests
+frontend-test:
+	@echo "Running frontend unit tests..."
+	cd $(FRONTEND_DIR) && npm run test:unit
+
+# Clean build artifacts and dependencies
+clean:
+	@echo "Cleaning build artifacts and dependencies..."
+	rm -rf $(FRONTEND_DIR)/node_modules
+	rm -rf $(FRONTEND_DIR)/dist
+
+# Start backend services
+up:
+	@echo "Starting backend services..."
+	docker compose -f $(COMPOSE_FILE) up -d
+
+# Stop backend services
+down:
+	@echo "Stopping backend services..."
+	docker compose -f $(COMPOSE_FILE) down
+
+# Build Docker image
+docker-build:
+	@echo "Building Docker image..."
+	docker compose -f $(COMPOSE_FILE) build
+
+# Restart backend services (no rebuild)
+restart:
+	@echo "Restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) restart
+
+# Rebuild and restart backend services
+refresh:
+	@echo "Rebuilding and restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) down
+	docker compose -f $(COMPOSE_FILE) build
+	docker compose -f $(COMPOSE_FILE) up -d

app.py

@@ -1,529 +0,0 @@
-"""Flask web application for book download service with URL rewrite support."""
-
-import logging
-import io, re, os
-import sqlite3
-from functools import wraps
-from flask import Flask, request, jsonify, render_template, send_file, send_from_directory
-from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.security import check_password_hash
-from werkzeug.wrappers import Response
-from flask import url_for as flask_url_for
-import typing
-
-from logger import setup_logger
-from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE
-from env import FLASK_HOST, FLASK_PORT, APP_ENV, CWA_DB_PATH, DEBUG
-import backend
-
-from models import SearchFilters
-
-logger = setup_logger(__name__)
-app = Flask(__name__)
-app.wsgi_app = ProxyFix(app.wsgi_app)  # type: ignore
-app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0  # Disable caching
-app.config['APPLICATION_ROOT'] = '/'
-
-# Flask logger
-app.logger.handlers = logger.handlers
-app.logger.setLevel(logger.level)
-# Also handle Werkzeug's logger
-werkzeug_logger = logging.getLogger('werkzeug')
-werkzeug_logger.handlers = logger.handlers
-werkzeug_logger.setLevel(logger.level)
-
-# Set up authentication defaults
-# The secret key will reset every time we restart, which will
-# require users to authenticate again
-app.config.update(
-    SECRET_KEY = os.urandom(64)
-)
-
-def login_required(f):
-    @wraps(f)
-    def decorated_function(*args, **kwargs):
-        # If the CWA_DB_PATH variable exists, but isn't a valid
-        # path, return a server error
-        if CWA_DB_PATH is not None and not os.path.isfile(CWA_DB_PATH):
-            logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path")
-            return Response("Internal Server Error", 500)
-        if not authenticate():
-            return Response(
-                response="Unauthorized",
-                status=401,
-                headers={
-                    "WWW-Authenticate": 'Basic realm="Calibre-Web-Automated-Book-Downloader"',
-                },
-            )
-        return f(*args, **kwargs)
-    return decorated_function
-
-def register_dual_routes(app : Flask) -> None:
-    """
-    Register each route both with and without the /request prefix.
-    This function should be called after all routes are defined.
-    """
-    # Store original url_map rules
-    rules = list(app.url_map.iter_rules())
-    
-    # Add /request prefix to each rule
-    for rule in rules:
-        if rule.rule != '/request/' and rule.rule != '/request':  # Skip if it's already a request route
-            # Create new routes with /request prefix, both with and without trailing slash
-            base_rule = rule.rule[:-1] if rule.rule.endswith('/') else rule.rule
-            if base_rule == '':  # Special case for root path
-                app.add_url_rule('/request', f"root_request", 
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-                app.add_url_rule('/request/', f"root_request_slash", 
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-            else:
-                app.add_url_rule(f"/request{base_rule}", 
-                               f"{rule.endpoint}_request",
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-                app.add_url_rule(f"/request{base_rule}/", 
-                               f"{rule.endpoint}_request_slash",
-                               view_func=app.view_functions[rule.endpoint],
-                               methods=rule.methods)
-    app.jinja_env.globals['url_for'] = url_for_with_request
-
-def url_for_with_request(endpoint : str, **values : typing.Any) -> str:
-    """Generate URLs with /request prefix by default."""
-    if endpoint == 'static':
-        # For static files, add /request prefix
-        url = flask_url_for(endpoint, **values)
-        return f"/request{url}"
-    return flask_url_for(endpoint, **values)
-
-@app.route('/')
-@login_required
-def index() -> str:
-    """
-    Render main page with search and status table.
-    """
-    return render_template('index.html', book_languages=_SUPPORTED_BOOK_LANGUAGE, default_language=BOOK_LANGUAGE, debug=DEBUG)
-
-@app.route('/favico<path:_>')
-@app.route('/request/favico<path:_>')
-@app.route('/request/static/favico<path:_>')
-def favicon(_ : typing.Any) -> Response:
-    return send_from_directory(os.path.join(app.root_path, 'static', 'media'),
-        'favicon.ico', mimetype='image/vnd.microsoft.icon')
-
-from typing import Union, Tuple
-
-if DEBUG:
-    import subprocess
-    import time
-    from cloudflare_bypasser import _reset_driver as STOP_GUI
-    @app.route('/debug', methods=['GET'])
-    @login_required
-    def debug() -> Union[Response, Tuple[Response, int]]:
-        """
-        This will run the /app/debug.sh script, which will generate a debug zip with all the logs
-        The file will be named /tmp/cwa-book-downloader-debug.zip
-        And then return it to the user
-        """
-        try:
-            # Run the debug script
-            STOP_GUI()
-            time.sleep(1)
-            result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True)
-            if result.returncode != 0:
-                raise Exception(f"Debug script failed: {result.stderr}")
-            logger.info(f"Debug script executed: {result.stdout}")
-            debug_file_path = result.stdout.strip().split('\n')[-1]
-            if not os.path.exists(debug_file_path):
-                logger.error("Debug zip file not found after running debug script")
-                return jsonify({"error": "Failed to generate debug information"}), 500
-                
-            # Return the file to the user
-            return send_file(
-                debug_file_path,
-                mimetype='application/zip',
-                download_name=os.path.basename(debug_file_path),
-                as_attachment=True
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}")
-            return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500
-        except Exception as e:
-            logger.error_trace(f"Debug endpoint error: {e}")
-            return jsonify({"error": str(e)}), 500
-
-if DEBUG:
-    @app.route('/api/restart', methods=['GET'])
-    @login_required
-    def restart() -> Union[Response, Tuple[Response, int]]:
-        """
-        Restart the application
-        """
-        os._exit(0)
-
-@app.route('/api/search', methods=['GET'])
-@login_required
-def api_search() -> Union[Response, Tuple[Response, int]]:
-    """
-    Search for books matching the provided query.
-
-    Query Parameters:
-        query (str): Search term (ISBN, title, author, etc.)
-        isbn (str): Book ISBN
-        author (str): Book Author
-        title (str): Book Title
-        lang (str): Book Language
-        sort (str): Order to sort results
-        content (str): Content type of book
-        format (str): File format filter (pdf, epub, mobi, azw3, fb2, djvu, cbz, cbr)
-
-    Returns:
-        flask.Response: JSON array of matching books or error response.
-    """
-    query = request.args.get('query', '')
-
-    filters = SearchFilters(
-        isbn = request.args.getlist('isbn'),
-        author = request.args.getlist('author'),
-        title = request.args.getlist('title'),
-        lang = request.args.getlist('lang'),
-        sort = request.args.get('sort'),
-        content = request.args.getlist('content'),
-        format = request.args.getlist('format'),
-    )
-
-    if not query and not any(vars(filters).values()):
-        return jsonify([])
-
-    try:
-        books = backend.search_books(query, filters)
-        return jsonify(books)
-    except Exception as e:
-        logger.error_trace(f"Search error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/info', methods=['GET'])
-@login_required
-def api_info() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get detailed book information.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON object with book details, or an error message.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        book = backend.get_book_info(book_id)
-        if book:
-            return jsonify(book)
-        return jsonify({"error": "Book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Info error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download', methods=['GET'])
-@login_required
-def api_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Queue a book for download.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: JSON status object indicating success or failure.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        priority = int(request.args.get('priority', 0))
-        success = backend.queue_book(book_id, priority)
-        if success:
-            return jsonify({"status": "queued", "priority": priority})
-        return jsonify({"error": "Failed to queue book"}), 500
-    except Exception as e:
-        logger.error_trace(f"Download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/status', methods=['GET'])
-@login_required
-def api_status() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current download queue status.
-
-    Returns:
-        flask.Response: JSON object with queue status.
-    """
-    try:
-        status = backend.queue_status()
-        return jsonify(status)
-    except Exception as e:
-        logger.error_trace(f"Status error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/localdownload', methods=['GET'])
-@login_required
-def api_local_download() -> Union[Response, Tuple[Response, int]]:
-    """
-    Download an EPUB file from local storage if available.
-
-    Query Parameters:
-        id (str): Book identifier (MD5 hash)
-
-    Returns:
-        flask.Response: The EPUB file if found, otherwise an error response.
-    """
-    book_id = request.args.get('id', '')
-    if not book_id:
-        return jsonify({"error": "No book ID provided"}), 400
-
-    try:
-        file_data, book_info = backend.get_book_data(book_id)
-        if file_data is None:
-            # Book data not found or not available
-            return jsonify({"error": "File not found"}), 404
-        # Santize the file name
-        file_name = book_info.title
-        file_name = re.sub(r'[\\/:*?"<>|]', '_', file_name.strip())[:245]
-        file_extension = book_info.format
-        # Prepare the file for sending to the client
-        data = io.BytesIO(file_data)
-        return send_file(
-            data,
-            download_name=f"{file_name}.{file_extension}",
-            as_attachment=True
-        )
-
-    except Exception as e:
-        logger.error_trace(f"Local download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/download/<book_id>/cancel', methods=['DELETE'])
-@login_required
-def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Cancel a download.
-
-    Path Parameters:
-        book_id (str): Book identifier to cancel
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        success = backend.cancel_download(book_id)
-        if success:
-            return jsonify({"status": "cancelled", "book_id": book_id})
-        return jsonify({"error": "Failed to cancel download or book not found"}), 404
-    except Exception as e:
-        logger.error_trace(f"Cancel download error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/<book_id>/priority', methods=['PUT'])
-@login_required
-def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]:
-    """
-    Set priority for a queued book.
-
-    Path Parameters:
-        book_id (str): Book identifier
-
-    Request Body:
-        priority (int): New priority level (lower number = higher priority)
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'priority' not in data:
-            return jsonify({"error": "Priority not provided"}), 400
-            
-        priority = int(data['priority'])
-        success = backend.set_book_priority(book_id, priority)
-        
-        if success:
-            return jsonify({"status": "updated", "book_id": book_id, "priority": priority})
-        return jsonify({"error": "Failed to update priority or book not found"}), 404
-    except ValueError:
-        return jsonify({"error": "Invalid priority value"}), 400
-    except Exception as e:
-        logger.error_trace(f"Set priority error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/reorder', methods=['POST'])
-@login_required
-def api_reorder_queue() -> Union[Response, Tuple[Response, int]]:
-    """
-    Bulk reorder queue by setting new priorities.
-
-    Request Body:
-        book_priorities (dict): Mapping of book_id to new priority
-
-    Returns:
-        flask.Response: JSON status indicating success or failure.
-    """
-    try:
-        data = request.get_json()
-        if not data or 'book_priorities' not in data:
-            return jsonify({"error": "book_priorities not provided"}), 400
-            
-        book_priorities = data['book_priorities']
-        if not isinstance(book_priorities, dict):
-            return jsonify({"error": "book_priorities must be a dictionary"}), 400
-            
-        # Validate all priorities are integers
-        for book_id, priority in book_priorities.items():
-            if not isinstance(priority, int):
-                return jsonify({"error": f"Invalid priority for book {book_id}"}), 400
-                
-        success = backend.reorder_queue(book_priorities)
-        
-        if success:
-            return jsonify({"status": "reordered", "updated_count": len(book_priorities)})
-        return jsonify({"error": "Failed to reorder queue"}), 500
-    except Exception as e:
-        logger.error_trace(f"Reorder queue error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/order', methods=['GET'])
-@login_required
-def api_queue_order() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get current queue order for display.
-
-    Returns:
-        flask.Response: JSON array of queued books with their order and priorities.
-    """
-    try:
-        queue_order = backend.get_queue_order()
-        return jsonify({"queue": queue_order})
-    except Exception as e:
-        logger.error_trace(f"Queue order error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/downloads/active', methods=['GET'])
-@login_required
-def api_active_downloads() -> Union[Response, Tuple[Response, int]]:
-    """
-    Get list of currently active downloads.
-
-    Returns:
-        flask.Response: JSON array of active download book IDs.
-    """
-    try:
-        active_downloads = backend.get_active_downloads()
-        return jsonify({"active_downloads": active_downloads})
-    except Exception as e:
-        logger.error_trace(f"Active downloads error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.route('/api/queue/clear', methods=['DELETE'])
-@login_required
-def api_clear_completed() -> Union[Response, Tuple[Response, int]]:
-    """
-    Clear all completed, errored, or cancelled books from tracking.
-
-    Returns:
-        flask.Response: JSON with count of removed books.
-    """
-    try:
-        removed_count = backend.clear_completed()
-        return jsonify({"status": "cleared", "removed_count": removed_count})
-    except Exception as e:
-        logger.error_trace(f"Clear completed error: {e}")
-        return jsonify({"error": str(e)}), 500
-
-@app.errorhandler(404)
-def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 404 (Not Found) errors.
-
-    Args:
-        error (HTTPException): The 404 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 404 status.
-    """
-    logger.warning(f"404 error: {request.url} : {error}")
-    return jsonify({"error": "Resource not found"}), 404
-
-@app.errorhandler(500)
-def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
-    """
-    Handle 500 (Internal Server) errors.
-
-    Args:
-        error (HTTPException): The 500 error raised by Flask.
-
-    Returns:
-        flask.Response: JSON error message with 500 status.
-    """
-    logger.error_trace(f"500 error: {error}")
-    return jsonify({"error": "Internal server error"}), 500
-
-def authenticate() -> bool:
-    """
-    Helper function that validates Basic credentials
-    against a Calibre-Web app.db SQLite database
-
-    Database structure:
-    - Table 'user' with columns: 'name' (username), 'password'
-    """
-
-    # If the database doesn't exist, the user is always authenticated
-    if not CWA_DB_PATH:
-        return True
-
-    # If no authorization object exists, return false to prompt
-    # a request to the user
-    if not request.authorization:
-        return False
-
-    username = request.authorization.get("username")
-    password = request.authorization.get("password")
-
-    # Validate credentials against database
-    try:
-        # Open database in true read-only mode to avoid journal/WAL writes on RO mounts
-        db_path = os.fspath(CWA_DB_PATH)
-        db_uri = f"file:{db_path}?mode=ro&immutable=1"
-        conn = sqlite3.connect(db_uri, uri=True)
-        cur = conn.cursor()
-        cur.execute("SELECT password FROM user WHERE name = ?", (username,))
-        row = cur.fetchone()
-        conn.close()
-
-        # Check if user exists and password is correct
-        if not row or not row[0] or not check_password_hash(row[0], password):
-            logger.error("User not found or password check failed")
-            return False
-
-    except Exception as e:
-        logger.error_trace(f"CWA DB or authentication send_from_directory: {e}")
-        return False
-
-    logger.info(f"Authentication successful for user {username}")
-    return True
-
-# Register all routes with /request prefix
-register_dual_routes(app)
-
-logger.log_resource_usage()
-
-if __name__ == '__main__':
-    logger.info(f"Starting Flask application on {FLASK_HOST}:{FLASK_PORT} IN {APP_ENV} mode")
-    app.run(
-        host=FLASK_HOST,
-        port=FLASK_PORT,
-        debug=DEBUG 
-    )

backend.py

@@ -1,338 +0,0 @@
-"""Backend logic for the book download application."""
-
-import threading, time
-import shutil
-from pathlib import Path
-from typing import Dict, List, Optional, Any, Tuple
-import subprocess
-import os
-from concurrent.futures import ThreadPoolExecutor, Future
-from threading import Event
-
-from logger import setup_logger
-from config import CUSTOM_SCRIPT
-from env import INGEST_DIR, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE, MAX_CONCURRENT_DOWNLOADS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL
-from models import book_queue, BookInfo, QueueStatus, SearchFilters
-import book_manager
-
-logger = setup_logger(__name__)
-
-def _sanitize_filename(filename: str) -> str:
-    """Sanitize a filename by replacing spaces with underscores and removing invalid characters."""
-    keepcharacters = (' ','.','_')
-    return "".join(c for c in filename if c.isalnum() or c in keepcharacters).rstrip()
-
-def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
-    """Search for books matching the query.
-    
-    Args:
-        query: Search term
-        filters: Search filters object
-        
-    Returns:
-        List[Dict]: List of book information dictionaries
-    """
-    try:
-        books = book_manager.search_books(query, filters)
-        return [_book_info_to_dict(book) for book in books]
-    except Exception as e:
-        logger.error_trace(f"Error searching books: {e}")
-        return []
-
-def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
-    """Get detailed information for a specific book.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Optional[Dict]: Book information dictionary if found
-    """
-    try:
-        book = book_manager.get_book_info(book_id)
-        return _book_info_to_dict(book)
-    except Exception as e:
-        logger.error_trace(f"Error getting book info: {e}")
-        return None
-
-def queue_book(book_id: str, priority: int = 0) -> bool:
-    """Add a book to the download queue with specified priority.
-    
-    Args:
-        book_id: Book identifier
-        priority: Priority level (lower number = higher priority)
-        
-    Returns:
-        bool: True if book was successfully queued
-    """
-    try:
-        book_info = book_manager.get_book_info(book_id)
-        book_queue.add(book_id, book_info, priority)
-        logger.info(f"Book queued with priority {priority}: {book_info.title}")
-        return True
-    except Exception as e:
-        logger.error_trace(f"Error queueing book: {e}")
-        return False
-
-def queue_status() -> Dict[str, Dict[str, Any]]:
-    """Get current status of the download queue.
-    
-    Returns:
-        Dict: Queue status organized by status type
-    """
-    status = book_queue.get_status()
-    # Convert Enum keys to strings and properly format the response
-    return {
-        status_type.value: books
-        for status_type, books in status.items()
-    }
-
-def get_book_data(book_id: str) -> Tuple[Optional[bytes], BookInfo]:
-    """Get book data for a specific book, including its title.
-    
-    Args:
-        book_id: Book identifier
-        
-    Returns:
-        Tuple[Optional[bytes], str]: Book data if available, and the book title
-    """
-    try:
-        book_info = book_queue._book_data[book_id]
-        path = book_info.download_path
-        with open(path, "rb") as f:
-            return f.read(), book_info
-    except Exception as e:
-        logger.error_trace(f"Error getting book data: {e}")
-        if book_info:
-            book_info.download_path = None
-        return None, book_info if book_info else BookInfo(id=book_id, title="Unknown")
-
-def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
-    """Convert BookInfo object to dictionary representation."""
-    return {
-        key: value for key, value in book.__dict__.items()
-        if value is not None
-    }
-
-def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Optional[str]:
-    """Download and process a book with cancellation support.
-    
-    Args:
-        book_id: Book identifier
-        cancel_flag: Threading event to signal cancellation
-        
-    Returns:
-        str: Path to the downloaded book if successful, None otherwise
-    """
-    try:
-        # Check for cancellation before starting
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before starting: {book_id}")
-            return None
-            
-        book_info = book_queue._book_data[book_id]
-        logger.info(f"Starting download: {book_info.title}")
-
-        if USE_BOOK_TITLE:
-            book_name = _sanitize_filename(book_info.title)
-        else:
-            book_name = book_id
-        book_name += f".{book_info.format}"
-        book_path = TMP_DIR / book_name
-
-        # Check cancellation before download
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before book manager call: {book_id}")
-            return None
-
-        # Update progress periodically during download
-        progress_thread = threading.Thread(
-            target=_update_download_progress,
-            args=(book_id, cancel_flag),
-            daemon=True
-        )
-        progress_thread.start()
-
-        success = book_manager.download_book(book_info, book_path)
-        
-        # Stop progress updates
-        cancel_flag.wait(0.1)  # Brief pause for progress thread cleanup
-        
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during download: {book_id}")
-            # Clean up partial download
-            if book_path.exists():
-                book_path.unlink()
-            return None
-            
-        if not success:
-            raise Exception("Unknown error downloading book")
-
-        # Check cancellation before post-processing
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled before post-processing: {book_id}")
-            if book_path.exists():
-                book_path.unlink()
-            return None
-
-        if CUSTOM_SCRIPT:
-            logger.info(f"Running custom script: {CUSTOM_SCRIPT}")
-            subprocess.run([CUSTOM_SCRIPT, book_path])
-            
-        intermediate_path = INGEST_DIR / f"{book_id}.crdownload"
-        final_path = INGEST_DIR / book_name
-        
-        if os.path.exists(book_path):
-            logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}")
-            try:
-                shutil.move(book_path, intermediate_path)
-            except Exception as e:
-                logger.debug(f"Error moving book: {e}, will try copying instead")
-                shutil.copy(book_path, intermediate_path)
-                os.remove(book_path)
-            
-            # Final cancellation check before completing
-            if cancel_flag.is_set():
-                logger.info(f"Download cancelled before final rename: {book_id}")
-                if intermediate_path.exists():
-                    intermediate_path.unlink()
-                return None
-                
-            os.rename(intermediate_path, final_path)
-            logger.info(f"Download completed successfully: {book_info.title}")
-            
-        return str(final_path)
-    except Exception as e:
-        if cancel_flag.is_set():
-            logger.info(f"Download cancelled during error handling: {book_id}")
-        else:
-            logger.error_trace(f"Error downloading book: {e}")
-        return None
-
-def _update_download_progress(book_id: str, cancel_flag: Event) -> None:
-    """Update download progress periodically."""
-    progress = 0.0
-    while not cancel_flag.is_set() and progress < 100.0:
-        # Simulate progress (in real implementation, this would get actual progress)
-        progress = min(100.0, progress + 10.0)
-        book_queue.update_progress(book_id, progress)
-        time.sleep(DOWNLOAD_PROGRESS_UPDATE_INTERVAL)
-
-def cancel_download(book_id: str) -> bool:
-    """Cancel a download.
-    
-    Args:
-        book_id: Book identifier to cancel
-        
-    Returns:
-        bool: True if cancellation was successful
-    """
-    return book_queue.cancel_download(book_id)
-
-def set_book_priority(book_id: str, priority: int) -> bool:
-    """Set priority for a queued book.
-    
-    Args:
-        book_id: Book identifier
-        priority: New priority level (lower = higher priority)
-        
-    Returns:
-        bool: True if priority was successfully changed
-    """
-    return book_queue.set_priority(book_id, priority)
-
-def reorder_queue(book_priorities: Dict[str, int]) -> bool:
-    """Bulk reorder queue.
-    
-    Args:
-        book_priorities: Dict mapping book_id to new priority
-        
-    Returns:
-        bool: True if reordering was successful
-    """
-    return book_queue.reorder_queue(book_priorities)
-
-def get_queue_order() -> List[Dict[str, any]]:
-    """Get current queue order for display."""
-    return book_queue.get_queue_order()
-
-def get_active_downloads() -> List[str]:
-    """Get list of currently active downloads."""
-    return book_queue.get_active_downloads()
-
-def clear_completed() -> int:
-    """Clear all completed downloads from tracking."""
-    return book_queue.clear_completed()
-
-def _process_single_download(book_id: str, cancel_flag: Event) -> None:
-    """Process a single download job."""
-    try:
-        book_queue.update_status(book_id, QueueStatus.DOWNLOADING)
-        download_path = _download_book_with_cancellation(book_id, cancel_flag)
-        
-        if cancel_flag.is_set():
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-            return
-            
-        if download_path:
-            book_queue.update_download_path(book_id, download_path)
-            new_status = QueueStatus.AVAILABLE
-        else:
-            new_status = QueueStatus.ERROR
-            
-        book_queue.update_status(book_id, new_status)
-        
-        logger.info(
-            f"Book {book_id} download {'successful' if download_path else 'failed'}"
-        )
-        
-    except Exception as e:
-        if not cancel_flag.is_set():
-            logger.error_trace(f"Error in download processing: {e}")
-            book_queue.update_status(book_id, QueueStatus.ERROR)
-        else:
-            logger.info(f"Download cancelled: {book_id}")
-            book_queue.update_status(book_id, QueueStatus.CANCELLED)
-
-def concurrent_download_loop() -> None:
-    """Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
-    logger.info(f"Starting concurrent download loop with {MAX_CONCURRENT_DOWNLOADS} workers")
-    
-    with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_DOWNLOADS, thread_name_prefix="BookDownload") as executor:
-        active_futures: Dict[Future, str] = {}  # Track active download futures
-        
-        while True:
-            # Clean up completed futures
-            completed_futures = [f for f in active_futures if f.done()]
-            for future in completed_futures:
-                book_id = active_futures.pop(future)
-                try:
-                    future.result()  # This will raise any exceptions from the worker
-                except Exception as e:
-                    logger.error_trace(f"Future exception for {book_id}: {e}")
-            
-            # Start new downloads if we have capacity
-            while len(active_futures) < MAX_CONCURRENT_DOWNLOADS:
-                next_download = book_queue.get_next()
-                if not next_download:
-                    break
-                    
-                book_id, cancel_flag = next_download
-                logger.info(f"Starting concurrent download: {book_id}")
-                
-                # Submit download job to thread pool
-                future = executor.submit(_process_single_download, book_id, cancel_flag)
-                active_futures[future] = book_id
-            
-            # Brief sleep to prevent busy waiting
-            time.sleep(MAIN_LOOP_SLEEP_TIME)
-
-# Start concurrent download coordinator
-download_coordinator_thread = threading.Thread(
-    target=concurrent_download_loop,
-    daemon=True,
-    name="DownloadCoordinator"
-)
-download_coordinator_thread.start()
-
-logger.info(f"Download system initialized with {MAX_CONCURRENT_DOWNLOADS} concurrent workers")

book_manager.py

@@ -1,381 +0,0 @@
-"""Book download manager handling search and retrieval operations."""
-
-import time, json, re
-from pathlib import Path
-from urllib.parse import quote
-from typing import List, Optional, Dict, Union
-from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet
-
-import downloader
-from logger import setup_logger
-from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL
-from env import AA_DONATOR_KEY, USE_CF_BYPASS, PRIORITIZE_WELIB
-from models import BookInfo, SearchFilters
-
-logger = setup_logger(__name__)
-
-
-def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
-    """Search for books matching the query.
-
-    Args:
-        query: Search term (ISBN, title, author, etc.)
-
-    Returns:
-        List[BookInfo]: List of matching books
-
-    Raises:
-        Exception: If no books found or parsing fails
-    """
-    query_html = quote(query)
-
-    if filters.isbn:
-        # ISBNs are included in query string
-        isbns = " || ".join(
-            [f"('isbn13:{isbn}' || 'isbn10:{isbn}')" for isbn in filters.isbn]
-        )
-        query_html = quote(f"({isbns}) {query}")
-
-    filters_query = ""
-
-    for value in filters.lang or BOOK_LANGUAGE:
-        if value != "all":
-            filters_query += f"&lang={quote(value)}"
-
-    if filters.sort:
-        filters_query += f"&sort={quote(filters.sort)}"
-
-    if filters.content:
-        for value in filters.content:
-            filters_query += f"&content={quote(value)}"
-
-    # Handle format filter
-    formats_to_use = filters.format if filters.format else SUPPORTED_FORMATS
-
-    index = 1
-    for filter_type, filter_values in vars(filters).items():
-        if filter_type == "author" or filter_type == "title" and filter_values:
-            for value in filter_values:
-                filters_query += (
-                    f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}"
-                )
-                index += 1
-
-    url = (
-        f"{AA_BASE_URL}"
-        f"/search?index=&page=1&display=table"
-        f"&acc=aa_download&acc=external_download"
-        f"&ext={'&ext='.join(formats_to_use)}"
-        f"&q={query_html}"
-        f"{filters_query}"
-    )
-
-    html = downloader.html_get_page(url)
-    if not html:
-        raise Exception("Failed to fetch search results")
-
-    if "No files found." in html:
-        logger.info(f"No books found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    soup = BeautifulSoup(html, "html.parser")
-    tbody: Tag | NavigableString | None = soup.find("table")
-
-    if not tbody:
-        logger.warning(f"No results table found for query: {query}")
-        raise Exception("No books found. Please try another query.")
-
-    books = []
-    if isinstance(tbody, Tag):
-        for line_tr in tbody.find_all("tr"):
-            try:
-                book = _parse_search_result_row(line_tr)
-                if book:
-                    books.append(book)
-            except Exception as e:
-                logger.error_trace(f"Failed to parse search result row: {e}")
-
-    books.sort(
-        key=lambda x: (
-            SUPPORTED_FORMATS.index(x.format)
-            if x.format in SUPPORTED_FORMATS
-            else len(SUPPORTED_FORMATS)
-        )
-    )
-
-    return books
-
-
-def _parse_search_result_row(row: Tag) -> Optional[BookInfo]:
-    """Parse a single search result row into a BookInfo object."""
-    try:
-        cells = row.find_all("td")
-        preview_img = cells[0].find("img")
-        preview = preview_img["src"] if preview_img else None
-
-        return BookInfo(
-            id=row.find_all("a")[0]["href"].split("/")[-1],
-            preview=preview,
-            title=cells[1].find("span").next,
-            author=cells[2].find("span").next,
-            publisher=cells[3].find("span").next,
-            year=cells[4].find("span").next,
-            language=cells[7].find("span").next,
-            format=cells[9].find("span").next.lower(),
-            size=cells[10].find("span").next,
-        )
-    except Exception as e:
-        logger.error_trace(f"Error parsing search result row: {e}")
-        return None
-
-
-def get_book_info(book_id: str) -> BookInfo:
-    """Get detailed information for a specific book.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-
-    Returns:
-        BookInfo: Detailed book information
-    """
-    url = f"{AA_BASE_URL}/md5/{book_id}"
-    html = downloader.html_get_page(url)
-
-    if not html:
-        raise Exception(f"Failed to fetch book info for ID: {book_id}")
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    return _parse_book_info_page(soup, book_id)
-
-
-def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
-    """Parse the book info page HTML into a BookInfo object."""
-    data = soup.select_one("body > main > div:nth-of-type(1)")
-
-    if not data:
-        raise Exception(f"Failed to parse book info for ID: {book_id}")
-
-    preview: str = ""
-
-    node = data.select_one("div:nth-of-type(1) > img")
-    if node:
-        preview_value = node.get("src", "")
-        if isinstance(preview_value, list):
-            preview = preview_value[0]
-        else:
-            preview = preview_value
-
-    data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
-    divs = list(data.children)
-    format = divs[13].text.split(" · ")[1].strip().lower()
-    size = divs[13].text.split(" · ")[2].strip().lower()
-
-    every_url = soup.find_all("a")
-    slow_urls_no_waitlist = set()
-    slow_urls_with_waitlist = set()
-    external_urls_libgen = set()
-    external_urls_z_lib = set()
-    external_urls_welib = set()
-
-    for url in every_url:
-        try:
-            if url.text.strip().lower().startswith("slow partner server"):
-                if (
-                    url.next is not None
-                    and url.next.next is not None
-                    and "waitlist" in url.next.next.strip().lower()
-                ):
-                    internal_text = url.next.next.strip().lower()
-                    if "no waitlist" in internal_text:
-                        slow_urls_no_waitlist.add(url["href"])
-                    else:
-                        slow_urls_with_waitlist.add(url["href"])
-            elif (
-                url.next is not None
-                and url.next.next is not None
-                and "click “GET” at the top" in url.next.next.text.strip()
-            ):
-                libgen_url = url["href"]
-                # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
-                libgen_url = libgen_url = re.sub(r'libgen\.(\w+)', 'libgen.gs', url["href"])
-                external_urls_libgen.add(libgen_url)
-            elif url.text.strip().lower().startswith("z-lib"):
-                if ".onion/" not in url["href"]:
-                    external_urls_z_lib.add(url["href"])
-        except:
-            pass
-
-    external_urls_welib = _get_download_urls_from_welib(book_id) if USE_CF_BYPASS else set()
-
-    urls = []
-    urls += list(external_urls_welib) if PRIORITIZE_WELIB else []
-    urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
-    urls += list(external_urls_libgen)
-    urls += list(external_urls_welib) if not PRIORITIZE_WELIB else []
-    urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
-    urls += list(external_urls_z_lib)
-
-    for i in range(len(urls)):
-        urls[i] = downloader.get_absolute_url(AA_BASE_URL, urls[i])
-
-    # Remove empty urls
-    urls = [url for url in urls if url != ""]
-
-    # Extract basic information
-    book_info = BookInfo(
-        id=book_id,
-        preview=preview,
-        title=divs[7].next.strip(),
-        publisher=divs[11].text.strip(),
-        author=divs[9].text.strip(),
-        format=format,
-        size=size,
-        download_urls=urls,
-    )
-
-    # Extract additional metadata
-    info = _extract_book_metadata(divs[-6])
-    book_info.info = info
-
-    # Set language and year from metadata if available
-    if info.get("Language"):
-        book_info.language = info["Language"][0]
-    if info.get("Year"):
-        book_info.year = info["Year"][0]
-
-    return book_info
-
-def _get_download_urls_from_welib(book_id: str) -> set[str]:
-    """Get download urls from welib.org."""
-    url = f"https://welib.org/md5/{book_id}"
-    logger.info(f"Getting download urls from welib.org for {book_id}. While this uses the bypasser, it will not start downloading them yet.")
-    html = downloader.html_get_page(url, use_bypasser=True)
-    if not html:
-        return []
-    soup = BeautifulSoup(html, "html.parser")
-    download_links = soup.find_all("a", href=True)
-    download_links = [link["href"] for link in download_links]
-    download_links = [link for link in download_links if "/slow_download/" in link]
-    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
-    return set(download_links)
-
-def _extract_book_metadata(
-    metadata_divs
-) -> Dict[str, List[str]]:
-    """Extract metadata from book info divs."""
-    info: Dict[str, List[str]] = {}
-
-    # Process the first set of metadata
-    sub_datas = metadata_divs.find_all("div")[0]
-    sub_datas = list(sub_datas.children)
-    for sub_data in sub_datas:
-        if sub_data.text.strip() == "":
-            continue
-        sub_data = list(sub_data.children)
-        key = sub_data[0].text.strip()
-        value = sub_data[1].text.strip()
-        if key not in info:
-            info[key] = set()
-        info[key].add(value)
-    
-    # make set into list
-    for key, value in info.items():
-        info[key] = list(value)
-
-    # Filter relevant metadata
-    relevant_prefixes = [
-        "ISBN-",
-        "ALTERNATIVE",
-        "ASIN",
-        "Goodreads",
-        "Language",
-        "Year",
-    ]
-    return {
-        k.strip(): v
-        for k, v in info.items()
-        if any(k.lower().startswith(prefix.lower()) for prefix in relevant_prefixes)
-        and "filename" not in k.lower()
-    }
-
-
-def download_book(book_info: BookInfo, book_path: Path) -> bool:
-    """Download a book from available sources.
-
-    Args:
-        book_id: Book identifier (MD5 hash)
-        title: Book title for logging
-
-    Returns:
-        Optional[BytesIO]: Book content buffer if successful
-    """
-
-    if len(book_info.download_urls) == 0:
-        book_info = get_book_info(book_info.id)
-    download_links = book_info.download_urls
-
-    # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources.
-    if AA_DONATOR_KEY != "":
-        download_links.insert(
-            0,
-            f"{AA_BASE_URL}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}",
-        )
-
-    for link in download_links:
-        try:
-            download_url = _get_download_url(link, book_info.title)
-            if download_url != "":
-                logger.info(f"Downloading `{book_info.title}` from `{download_url}`")
-                data = downloader.download_url(download_url, book_info.size or "")
-                if not data:
-                    raise Exception("No data received")
-
-                logger.info(f"Download finished. Writing to {book_path}")
-                with open(book_path, "wb") as f:
-                    f.write(data.getbuffer())
-                logger.info(f"Writing `{book_info.title}` successfully")
-                return True
-
-        except Exception as e:
-            logger.error_trace(f"Failed to download from {link}: {e}")
-            continue
-
-    return False
-
-
-def _get_download_url(link: str, title: str) -> str:
-    """Extract actual download URL from various source pages."""
-
-    url = ""
-
-    if link.startswith(f"{AA_BASE_URL}/dyn/api/fast_download.json"):
-        page = downloader.html_get_page(link)
-        url = json.loads(page).get("download_url")
-    else:
-        html = downloader.html_get_page(link)
-
-        if html == "":
-            return ""
-
-        soup = BeautifulSoup(html, "html.parser")
-
-        if link.startswith("https://z-lib."):
-            download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
-            if download_link:
-                url = download_link[0]["href"]
-        elif "/slow_download/" in link:
-            download_links = soup.find_all("a", href=True, string="📚 Download now")
-            if not download_links:
-                countdown = soup.find_all("span", class_="js-partner-countdown")
-                if countdown:
-                    sleep_time = int(countdown[0].text)
-                    logger.info(f"Waiting {sleep_time}s for {title}")
-                    time.sleep(sleep_time)
-                    url = _get_download_url(link, title)
-            else:
-                url = download_links[0]["href"]
-        else:
-            url = soup.find_all("a", string="GET")[0]["href"]
-
-    return downloader.get_absolute_url(link, url)

cloudflare_bypasser.py

@@ -1,478 +0,0 @@
-import time
-import os
-import socket
-from urllib.parse import urlparse
-import threading
-import env
-from env import LOG_DIR, DEBUG
-import signal
-from datetime import datetime
-import subprocess
-
-# --- SeleniumBase Import ---
-from seleniumbase import Driver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.common.exceptions import TimeoutException
-
-import network
-from logger import setup_logger
-from env import MAX_RETRY, DEFAULT_SLEEP
-from config import PROXIES, CUSTOM_DNS, DOH_SERVER, VIRTUAL_SCREEN_SIZE, RECORDING_DIR
-
-logger = setup_logger(__name__)
-network.init()
-
-DRIVER = None
-DISPLAY = {
-    "xvfb": None,
-    "ffmpeg": None,
-}
-LAST_USED = None
-LOCKED = threading.Lock()
-TENTATIVE_CURRENT_URL = None
-
-def _reset_pyautogui_display_state():
-    try:
-        import pyautogui
-        import Xlib.display
-        pyautogui._pyautogui_x11._display = (
-                    Xlib.display.Display(os.environ['DISPLAY'])
-                )
-    except Exception as e:
-        logger.warning(f"Error resetting pyautogui display state: {e}")
-
-def _is_bypassed(sb) -> bool:
-    """Enhanced bypass detection with more comprehensive checks"""
-    try:
-        # Get page information with error handling
-        try:
-            title = sb.get_title().lower()
-        except:
-            title = ""
-            
-        try:
-            body = sb.get_text("body").lower()
-        except:
-            body = ""
-            
-        try:
-            current_url = sb.get_current_url()
-        except:
-            current_url = ""
-        
-        # Enhanced verification texts for newer Cloudflare versions
-        verification_texts = [
-            "just a moment",
-            "verify you are human",
-            "verifying you are human",
-            "needs to review the security of your connection before proceeding",
-            "checking your browser",
-            "checking connection",
-            "attention required",
-            "access denied",
-            "needs to review the security of your connection",
-            "checking the site connection security",
-            "enable javascript and cookies to continue",
-            "ray id",
-            "cloudflare",
-            "please wait",
-            "ddos protection",
-            "security check",
-            "browser check",
-            "moment please",
-            "hold on",
-            "loading",
-            "one more step",
-            "challenge"
-        ]
-        
-        # Check for Cloudflare indicators
-        for text in verification_texts:
-            if text in title or text in body:
-                logger.debug(f"Cloudflare indicator found: '{text}' in page")
-                return False
-        
-        # Additional checks for specific Cloudflare patterns
-        if "cf-" in body or "cloudflare" in current_url.lower():
-            logger.debug("Cloudflare patterns detected in page")
-            return False
-            
-        # Check if we're still on a challenge page (common Cloudflare pattern)
-        if "/cdn-cgi/" in current_url:
-            logger.debug("Still on Cloudflare CDN challenge page")
-            return False
-            
-        # If page is mostly empty, it might still be loading
-        if len(body.strip()) < 50:
-            logger.debug("Page content too short, might still be loading")
-            return False
-            
-        logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {len(body)}")
-        return True
-        
-    except Exception as e:
-        logger.warning(f"Error checking bypass status: {e}")
-        # If we can't check, assume we're not bypassed
-        return False
-
-def _bypass_method_1(sb) -> bool:
-    """Original bypass method using uc_gui_click_captcha"""
-    try:
-        logger.debug("Attempting bypass method 1: uc_gui_click_captcha")
-        sb.uc_gui_click_captcha()
-        time.sleep(3)
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 1 failed on first try: {e}")
-        try:
-            time.sleep(5)
-            sb.wait_for_element_visible('body', timeout=10)
-            sb.uc_gui_click_captcha()
-            time.sleep(3)
-            return _is_bypassed(sb)
-        except Exception as e2:
-            logger.debug(f"Method 1 failed on second try: {e2}")
-            try:
-                time.sleep(DEFAULT_SLEEP)
-                sb.uc_gui_click_captcha()
-                time.sleep(5)
-                return _is_bypassed(sb)
-            except Exception as e3:
-                logger.debug(f"Method 1 completely failed: {e3}")
-                return False
-
-def _bypass_method_2(sb) -> bool:
-    """Alternative bypass method using longer waits and manual interaction"""
-    try:
-        logger.debug("Attempting bypass method 2: wait and reload")
-        # Wait longer for page to load completely
-        time.sleep(10)
-        
-        # Try refreshing the page
-        sb.refresh()
-        time.sleep(8)
-        
-        # Check if bypass worked after refresh
-        if _is_bypassed(sb):
-            return True
-            
-        # Try clicking on the page center (sometimes helps trigger bypass)
-        try:
-            sb.click_if_visible("body", timeout=5)
-            time.sleep(5)
-        except:
-            pass
-            
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 2 failed: {e}")
-        return False
-
-def _bypass_method_3(sb) -> bool:
-    """Third bypass method using user-agent rotation and stealth mode"""
-    try:
-        logger.debug("Attempting bypass method 3: stealth approach")
-        # Wait a random amount to appear more human
-        import random
-        wait_time = random.uniform(8, 15)
-        time.sleep(wait_time)
-        
-        # Try to scroll the page (human-like behavior)
-        try:
-            sb.scroll_to_bottom()
-            time.sleep(2)
-            sb.scroll_to_top()
-            time.sleep(3)
-        except:
-            pass
-            
-        # Check if this helped
-        if _is_bypassed(sb):
-            return True
-            
-        # Try the original captcha click as last resort
-        try:
-            sb.uc_gui_click_captcha()
-            time.sleep(5)
-        except:
-            pass
-            
-        return _is_bypassed(sb)
-    except Exception as e:
-        logger.debug(f"Method 3 failed: {e}")
-        return False
-
-def _bypass(sb, max_retries: int = MAX_RETRY) -> None:
-    """Enhanced bypass function with multiple strategies"""
-    try_count = 0
-    methods = [_bypass_method_1, _bypass_method_2, _bypass_method_3]
-
-    while not _is_bypassed(sb):
-        if try_count >= max_retries:
-            logger.warning("Exceeded maximum retries. Bypass failed.")
-            break
-            
-        method_index = try_count % len(methods)
-        method = methods[method_index]
-        
-        logger.info(f"Bypass attempt {try_count + 1} / {max_retries} using {method.__name__}")
-        
-        try_count += 1
-
-        # Progressive backoff: wait longer between retries
-        wait_time = min(DEFAULT_SLEEP * (try_count - 1), 15)
-        if wait_time > 0:
-            logger.info(f"Waiting {wait_time}s before trying...")
-            time.sleep(wait_time)
-
-        try:
-            if method(sb):
-                logger.info(f"Bypass successful using {method.__name__}")
-                return
-        except Exception as e:
-            logger.warning(f"Exception in {method.__name__}: {e}")
-
-        logger.info(f"Bypass method {method.__name__} failed.")
-
-def _get_chromium_args():
-    
-    arguments = [
-        # Ignore certificate and SSL errors (similar to curl's --insecure)
-        "--ignore-certificate-errors",
-        "--ignore-ssl-errors",
-        "--allow-running-insecure-content",
-        "--ignore-certificate-errors-spki-list",
-        "--ignore-certificate-errors-skip-list"
-    ]
-    
-    # Conditionally add verbose logging arguments
-    if DEBUG:
-        arguments.extend([
-            "--enable-logging", # Enable Chrome browser logging
-            "--v=1",            # Set verbosity level for Chrome logs
-            "--log-file=" + str(LOG_DIR / "chrome_browser.log")
-        ])
-
-    # Add proxy settings if configured
-    if PROXIES:
-        proxy_url = PROXIES.get('https') or PROXIES.get('http')
-        if proxy_url:
-            arguments.append(f'--proxy-server={proxy_url}')
-
-    # --- Add Custom DNS settings ---
-    try:
-        if len(CUSTOM_DNS) > 0:
-            if DOH_SERVER:
-                logger.info(f"Configuring DNS over HTTPS (DoH) with server: {DOH_SERVER}")
-
-                # TODO: This is probably broken and a halucination,
-                # but it should still default to google DOH so its fine...
-                arguments.extend(['--enable-features=DnsOverHttps', '--dns-over-https-mode=secure', f'--dns-over-https-servers="{DOH_SERVER}"'])
-                doh_hostname = urlparse(DOH_SERVER).hostname
-                if doh_hostname:
-                    try:
-                        arguments.append(f'--host-resolver-rules=MAP {doh_hostname} {socket.gethostbyname(doh_hostname)}')
-                    except socket.gaierror:
-                        logger.warning(f"Could not resolve DoH hostname: {doh_hostname}")
-            elif CUSTOM_DNS:
-                resolver_rules = [f"MAP * {dns_server}" for dns_server in CUSTOM_DNS]
-                if resolver_rules:
-                    arguments.append(f'--host-resolver-rules={",".join(resolver_rules)}') 
-    except Exception as e:
-        logger.error_trace(f"Error configuring DNS settings: {e}")
-    return arguments
-
-CHROMIUM_ARGS = _get_chromium_args()
-
-def _get(url, retry : int = MAX_RETRY):
-    try:
-        logger.info(f"SB_GET: {url}")
-        sb = _get_driver()
-        
-        # Enhanced page loading with better error handling
-        logger.debug("Opening URL with SeleniumBase...")
-        sb.uc_open_with_reconnect(url, DEFAULT_SLEEP)
-        time.sleep(DEFAULT_SLEEP)
-        
-        # Log current page title and URL for debugging
-        try:
-            current_url = sb.get_current_url()
-            current_title = sb.get_title()
-            logger.debug(f"Page loaded - URL: {current_url}, Title: {current_title}")
-        except Exception as debug_e:
-            logger.debug(f"Could not get page info: {debug_e}")
-        
-        # Attempt bypass
-        logger.debug("Starting bypass process...")
-        _bypass(sb)
-        
-        if _is_bypassed(sb):
-            logger.info("Bypass successful.")
-            return sb.page_source
-        else:
-            logger.warning("Bypass completed but page still shows Cloudflare protection")
-            # Log page content for debugging (truncated)
-            try:
-                page_text = sb.get_text("body")[:500] + "..." if len(sb.get_text("body")) > 500 else sb.get_text("body")
-                logger.debug(f"Page content: {page_text}")
-            except:
-                pass
-            
-    except Exception as e:
-        # Enhanced error logging with full stack trace
-        import traceback
-        error_details = f"Exception type: {type(e).__name__}, Message: {str(e)}"
-        stack_trace = traceback.format_exc()
-        
-        if retry == 0:
-            logger.error(f"Failed to initialize browser after all retries: {error_details}")
-            logger.debug(f"Full stack trace: {stack_trace}")
-            _reset_driver()
-            raise e
-        
-        logger.warning(f"Failed to bypass Cloudflare (retry {MAX_RETRY - retry + 1}/{MAX_RETRY}): {error_details}")
-        logger.debug(f"Stack trace: {stack_trace}")
-        
-        # Reset driver on certain errors
-        if "WebDriverException" in str(type(e)) or "SessionNotCreatedException" in str(type(e)):
-            logger.info("Resetting driver due to WebDriver error...")
-            _reset_driver()
-            
-    return _get(url, retry - 1)
-
-def get(url, retry : int = MAX_RETRY):
-    global LOCKED, TENTATIVE_CURRENT_URL, LAST_USED
-    with LOCKED:
-        TENTATIVE_CURRENT_URL = url
-        ret = _get(url, retry)
-        LAST_USED = time.time()
-        return ret
-
-def _init_driver():
-    global DRIVER
-    if DRIVER:
-        _reset_driver()
-    driver = Driver(uc=True, headless=False, size=f"{VIRTUAL_SCREEN_SIZE[0]},{VIRTUAL_SCREEN_SIZE[1]}", chromium_arg=CHROMIUM_ARGS)
-    DRIVER = driver
-    time.sleep(DEFAULT_SLEEP)
-    return driver
-
-def _get_driver():
-    global DRIVER, DISPLAY
-    global LAST_USED
-    logger.info("Getting driver...")
-    LAST_USED = time.time()
-    if env.DOCKERMODE and env.USE_CF_BYPASS and not DISPLAY["xvfb"]:
-        from pyvirtualdisplay import Display
-        display = Display(visible=False, size=VIRTUAL_SCREEN_SIZE)
-        display.start()
-        logger.info("Display started")
-        DISPLAY["xvfb"] = display
-        time.sleep(DEFAULT_SLEEP)
-        _reset_pyautogui_display_state()
-
-        if env.DEBUG:
-            timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
-            output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
-
-            ffmpeg_cmd = [
-                "ffmpeg",
-                "-y",
-                "-f", "x11grab",
-                "-video_size", f"{VIRTUAL_SCREEN_SIZE[0]}x{VIRTUAL_SCREEN_SIZE[1]}",
-                "-i", f":{display.display}",
-                "-c:v", "libx264",
-                "-preset", "ultrafast",  # or "veryfast" (trade speed for slightly better compression)
-                "-maxrate", "700k",      # Slightly higher bitrate for text clarity
-                "-bufsize", "1400k",    # Buffer size (2x maxrate)
-                "-crf", "36",  # Adjust as needed:  higher = smaller, lower = better quality (23 is visually lossless)
-                "-pix_fmt", "yuv420p",  # Crucial for compatibility with most players
-                "-tune", "animation",   # Optimize encoding for screen content
-                "-x264-params", "bframes=0:deblock=-1,-1", # Optimize for text, disable b-frames and deblocking
-                "-r", "15",         # Reduce frame rate (if content allows)
-                "-an",                # Disable audio recording (if not needed)
-                output_file.as_posix(),
-                "-nostats", "-loglevel", "0"
-            ]
-            logger.info("Starting FFmpeg recording to %s", output_file)
-            logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
-            DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
-    if not DRIVER:
-        return _init_driver()
-    logger.log_resource_usage()
-    return DRIVER
-
-def _reset_driver():
-    logger.log_resource_usage()
-    logger.info("Resetting driver...")
-    global DRIVER, DISPLAY
-    if DRIVER:
-        try:
-            DRIVER.quit()
-            DRIVER = None
-        except Exception as e:
-            logger.warning(f"Error quitting driver: {e}")
-        time.sleep(0.5)
-    if DISPLAY["xvfb"]:
-        try:
-            DISPLAY["xvfb"].stop()
-            DISPLAY["xvfb"] = None
-        except Exception as e:
-            logger.warning(f"Error stopping display: {e}")
-        time.sleep(0.5)
-    try:
-        os.system("pkill -f Xvfb")
-    except Exception as e:
-        logger.debug(f"Error killing Xvfb: {e}")
-    time.sleep(0.5)
-    if DISPLAY["ffmpeg"]:
-        try:
-            DISPLAY["ffmpeg"].send_signal(signal.SIGINT)
-            DISPLAY["ffmpeg"] = None
-        except Exception as e:
-            logger.debug(f"Error stopping ffmpeg: {e}")
-        time.sleep(0.5)
-    try:
-        os.system("pkill -f ffmpeg")
-    except Exception as e:
-        logger.debug(f"Error killing ffmpeg: {e}")
-    time.sleep(0.5)
-    try:
-        os.system("pkill -f chrom")
-    except Exception as e:
-        logger.debug(f"Error killing chrom: {e}")
-    time.sleep(0.5)
-    logger.info("Driver reset.")
-    logger.log_resource_usage()
-
-def _cleanup_driver():
-    global LOCKED
-    global LAST_USED
-    with LOCKED:
-        if LAST_USED:
-            if time.time() - LAST_USED >= env.BYPASS_RELEASE_INACTIVE_MIN * 60:
-                _reset_driver()
-                LAST_USED = None
-                logger.info("Driver reset due to inactivity.")
-
-def _cleanup_loop():
-    while True:
-        _cleanup_driver()
-        time.sleep(max(env.BYPASS_RELEASE_INACTIVE_MIN / 2, 1))
-
-def _init_cleanup_thread():
-    cleanup_thread = threading.Thread(target=_cleanup_loop)
-    cleanup_thread.daemon = True
-    cleanup_thread.start()
-
-def wait_for_result(func, timeout : int = 10, condition : any = True):
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        result = func()
-        if condition(result):
-            return result
-        time.sleep(0.5)
-    return None
-_init_cleanup_thread()

compose/docker-compose.lite.yml

@@ -0,0 +1,16 @@
+services:
+  shelfmark-lite:
+    image: ghcr.io/calibrain/shelfmark-lite:latest
+    container_name: shelfmark-lite
+    environment:
+      # EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.tor.yml

@@ -0,0 +1,20 @@
+# Routes all traffic through Tor - requires NET_ADMIN capability
+services:
+  shelfmark-tor:
+    image: ghcr.io/calibrain/shelfmark:latest
+    environment:
+      FLASK_PORT: 8084
+      USING_TOR: true
+      PUID: 1000
+      PGID: 1000
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    container_name: shelfmark
+    environment:
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

config.py

@@ -1,99 +0,0 @@
-"""Configuration settings for the book downloader application."""
-
-import os
-from pathlib import Path
-import json
-import env
-from logger import setup_logger
-
-logger = setup_logger(__name__)
-
-for key, value in env.__dict__.items():
-    if not key.startswith('_'):
-        if key == "AA_DONATOR_KEY" and value.strip() != "":
-            value = "REDACTED"
-        logger.info(f"{key}: {value}")
-
-with open("data/book-languages.json") as file:
-    _SUPPORTED_BOOK_LANGUAGE = json.load(file)
-
-# Directory settings
-BASE_DIR = Path(__file__).resolve().parent
-logger.info(f"BASE_DIR: {BASE_DIR}")
-if env.ENABLE_LOGGING:
-    env.LOG_DIR.mkdir(exist_ok=True)
-
-# Create necessary directories
-env.TMP_DIR.mkdir(exist_ok=True)
-env.INGEST_DIR.mkdir(exist_ok=True)
-
-CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
-logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
-logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
-logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
-
-# Network settings
-_custom_dns = env._CUSTOM_DNS.lower().strip()
-_doh_server = ""
-if _custom_dns == "google":
-    CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860::8888", "2001:4860:4860::8844"]
-    _doh_server = "https://dns.google/dns-query"
-elif _custom_dns == "quad9":
-    CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:fe::fe", "26620:fe::9"]
-    _doh_server = "https://dns.quad9.net/dns-query"
-elif _custom_dns == "cloudflare":
-    CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700::1111", "2606:4700:4700::1001"]
-    _doh_server = "https://cloudflare-dns.com/dns-query"
-elif _custom_dns == "opendns":
-    CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:119:35::35", "2620:119:53::53"]
-    _doh_server = "https://doh.opendns.com/dns-query"
-else:
-    _custom_dns_ip = _custom_dns.split(",")
-    CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()]
-logger.info(f"CUSTOM_DNS: {CUSTOM_DNS}")
-DOH_SERVER = _doh_server
-if env.USE_DOH:
-    DOH_SERVER = _doh_server
-else:
-    DOH_SERVER = ""
-logger.info(f"DOH_SERVER: {DOH_SERVER}")
-
-# Proxy settings
-PROXIES = {}
-if env.HTTP_PROXY:
-    PROXIES["http"] = env.HTTP_PROXY
-if env.HTTPS_PROXY:
-    PROXIES["https"] = env.HTTPS_PROXY
-logger.info(f"PROXIES: {PROXIES}")
-
-# Anna's Archive settings
-AA_BASE_URL = env._AA_BASE_URL
-AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
-AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
-AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
-
-# File format settings
-SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
-logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
-
-# Complex language processing logic kept in config.py
-BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
-BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
-if len(BOOK_LANGUAGE) == 0:
-    BOOK_LANGUAGE = ['en']
-
-# Custom script settings with validation logic
-CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
-if CUSTOM_SCRIPT:
-    if not os.path.exists(CUSTOM_SCRIPT):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
-        CUSTOM_SCRIPT = ""
-    elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
-        CUSTOM_SCRIPT = ""
-
-# Debugging settings
-VIRTUAL_SCREEN_SIZE = (1024, 768)
-RECORDING_DIR = env.LOG_DIR / "recording"
-if env.DEBUG:
-    RECORDING_DIR.mkdir(parents=True, exist_ok=True)

docker-compose.bypass-test.yml

@@ -0,0 +1,39 @@
+# Bypass testing - switch between dev build and v1.0.1
+# Usage:
+#   Test dev build:  docker compose -f docker-compose.bypass-test.yml up shelfmark-dev
+#   Test v1.0.1:     docker compose -f docker-compose.bypass-test.yml up shelfmark-stable
+#   Pull latest dev: docker compose -f docker-compose.bypass-test.yml build shelfmark-dev
+#   Pull v1.0.1:     docker compose -f docker-compose.bypass-test.yml pull shelfmark-stable
+
+services:
+  # Dev image from registry
+  shelfmark-dev:
+    image: ghcr.io/calibrain/shelfmark:dev
+    container_name: shelfmark-bypass-dev
+    environment:
+      PUID: 1000
+      PGID: 1000
+      DEBUG: true
+    ports:
+      - 8084:8084
+    volumes:
+      - ./.local/bypass-test/config-dev:/config
+      - ./.local/bypass-test/books:/books
+      - ./.local/bypass-test/log-dev:/var/log/shelfmark
+      - ./.local/bypass-test/tmp:/tmp/shelfmark
+
+  # Stable v1.0.1 for comparison
+  shelfmark-stable:
+    image: ghcr.io/calibrain/shelfmark:1.0.1
+    container_name: shelfmark-bypass-stable
+    environment:
+      PUID: 1000
+      PGID: 1000
+      DEBUG: true
+    ports:
+      - 8085:8084
+    volumes:
+      - ./.local/bypass-test/config-stable:/config
+      - ./.local/bypass-test/books:/books
+      - ./.local/bypass-test/log-stable:/var/log/shelfmark
+      - ./.local/bypass-test/tmp:/tmp/shelfmark

docker-compose.dev.extbp.yml

@@ -0,0 +1,25 @@
+# Local development - External bypasser variant (lite)
+services:
+  shelfmark-lite-dev:
+    extends:
+      file: ./compose/docker-compose.lite.yml
+      service: shelfmark-lite
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark-lite
+    environment:
+      DEBUG: true
+      EXT_BYPASSER_URL: http://flaresolverr:8191
+      EXT_BYPASSER_PATH: /v1
+      EXT_BYPASSER_TIMEOUT: 60000
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads
+
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.dev.tor.yml

@@ -0,0 +1,20 @@
+# Local development - Tor variant
+services:
+  shelfmark-tor-dev:
+    extends:
+      file: ./compose/docker-compose.tor.yml
+      service: shelfmark-tor
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    environment:
+      DEBUG: true
+      USING_TOR: true
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.dev.yml

@@ -1,17 +1,24 @@
+# Local development - builds from source with debug enabled
 services:
-  calibre-web-automated-book-downloader-dev:
+  shelfmark-dev:
     extends:
-      file: ./docker-compose.yml
-      service: calibre-web-automated-book-downloader
+      file: ./compose/docker-compose.yml
+      service: shelfmark
     build:
       context: .
       dockerfile: Dockerfile
-      target: cwa-bd
+      target: shelfmark
+    cap_add:
+      - SYS_PTRACE
     environment:
       DEBUG: true
-      APP_ENV: dev
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
+      # HIDE_LOCAL_AUTH: true
+      OIDC_AUTO_REDIRECT: true
     volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      - ./shelfmark:/app/shelfmark:ro
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.test-clients.yml

@@ -0,0 +1,181 @@
+# Test stack for download client development
+# Includes shelfmark + all download clients on same network with shared volumes
+#
+# Usage:
+#   docker compose -f docker-compose.test-clients.yml up -d
+#   # Access shelfmark at http://localhost:8084
+#   # Configure clients in Settings > Prowlarr > Download Clients
+#
+# Web UIs:
+#   - shelfmark:    http://localhost:8084
+#   - Prowlarr:     http://localhost:9696 (no auth by default)
+#   - qBittorrent:  http://localhost:8080 (check container logs for temp password)
+#   - Transmission: http://localhost:9091 (admin / admin)
+#   - Deluge:       http://localhost:8112 (password: deluge)
+#   - NZBGet:       http://localhost:6789 (nzbget / tegbzn6789)
+#   - SABnzbd:      http://localhost:8085 (complete setup wizard for API key)
+#   - rTorrent:     http://localhost:8000 (admin / admin - if auth enabled)
+#
+
+services:
+  shelfmark:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    container_name: test-shelfmark
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      # All client configuration is done via Settings UI
+      # Use Docker service names for URLs:
+      #   - qBittorrent: http://qbittorrent:8080
+      #   - Transmission: http://transmission:9091
+      #   - Deluge Web UI: http://deluge:8112
+      #   - NZBGet: http://nzbget:6789
+      #   - SABnzbd: http://sabnzbd:8080
+      #   - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
+    ports:
+      - "8084:8084"
+    volumes:
+      # Config and state
+      - ./.local/test-clients/shelfmark/config:/config
+      - ./.local/test-clients/shelfmark/log:/var/log/shelfmark
+      # Book destination directory (where completed books go)
+      - ./.local/test-clients/books:/books
+      # Staging directory
+      - ./.local/test-clients/tmp:/tmp/shelfmark
+      # CRITICAL: Mount client download directories so shelfmark can access completed files
+      - ./.local/test-clients/downloads:/downloads
+      # Mount source code for hot-reload (no rebuild needed for Python changes)
+      - ./shelfmark:/app/shelfmark:ro
+      # Mount tests for running pytest in container
+      - ./tests:/app/tests:ro
+      - ./pyproject.toml:/app/pyproject.toml:ro
+      # Mount client configs for integration tests to read credentials
+      - ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
+      - ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
+    depends_on:
+      - nzbget
+      - sabnzbd
+      - qbittorrent
+      - transmission
+      - deluge
+      - rtorrent
+    restart: unless-stopped
+
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:latest
+    container_name: test-prowlarr
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/prowlarr/config:/config
+    ports:
+      - "9696:9696"
+    restart: unless-stopped
+
+  nzbget:
+    image: lscr.io/linuxserver/nzbget:latest
+    container_name: test-nzbget
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/nzbget/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/nzbget/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "6789:6789"  # Web UI / JSON-RPC
+    restart: unless-stopped
+
+  sabnzbd:
+    image: lscr.io/linuxserver/sabnzbd:latest
+    container_name: test-sabnzbd
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/sabnzbd/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8085:8080"  # Web UI (external:internal)
+    restart: unless-stopped
+
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: test-qbittorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - WEBUI_PORT=8080
+    volumes:
+      - ./.local/test-clients/qbittorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/qbittorrent/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "8080:8080"  # Web UI / API
+      - "6882:6881"
+      - "6882:6881/udp"
+    restart: unless-stopped
+
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: test-transmission
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - USER=admin
+      - PASS=admin
+    volumes:
+      - ./.local/test-clients/transmission/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "9091:9091"  # Web UI / RPC
+      - "51413:51413"
+      - "51413:51413/udp"
+    restart: unless-stopped
+
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: test-deluge
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - DELUGE_LOGLEVEL=error
+    volumes:
+      - ./.local/test-clients/deluge/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8112:8112"  # Web UI
+      - "58846:58846"  # Daemon RPC
+      - "6881:6881"
+      - "6881:6881/udp"
+    restart: unless-stopped
+
+  rtorrent:
+    image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
+    container_name: test-rtorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/rtorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8000:8000"  # XMLRPC
+      - "8089:8080"  # ruTorrent Web UI
+      - "9000:9000"  # SCGI port
+      - "50000:50000"  # Incoming connections
+      - "6881:6881/udp"
+    restart: unless-stopped

docker-compose.tor.dev.yml

@@ -1,15 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor-dev:
-    extends:
-      file: ./docker-compose.tor.yml
-      service: calibre-web-automated-book-downloader-tor
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-tor
-    environment:
-      DEBUG: true
-      APP_ENV: dev
-    volumes:
-      - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
-      - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader

docker-compose.tor.yml

@@ -1,21 +0,0 @@
-services:
-  calibre-web-automated-book-downloader-tor:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      USING_TOR: true
-      APP_ENV: prod
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-    # This is where the books will be downloaded to, usually it would be 
-    # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest

docker-compose.yml

@@ -1,30 +0,0 @@
-services:
-  calibre-web-automated-book-downloader:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
-    # Uncomment to build the image from the Dockerfile for local testing changes.
-    # Remember to comment out the image line above.
-    #build: .
-    container_name: calibre-web-automated-book-downloader
-    environment:
-      FLASK_PORT: 8084
-      LOG_LEVEL: info
-      BOOK_LANGUAGE: en
-      USE_BOOK_TITLE: true
-      TZ: America/New_York
-      APP_ENV: prod
-      UID: 1000
-      GID: 100
-      # CWA_DB_PATH: /auth/app.db  # Comment out to disable authentication
-      # Queue management settings
-      MAX_CONCURRENT_DOWNLOADS: 3
-      DOWNLOAD_PROGRESS_UPDATE_INTERVAL: 5
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      # This is where the books will be downloaded to, usually it would be
-      # the same as whatever you gave in "calibre-web-automated"
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      # This is the location of CWA's app.db, which contains authentication
-      # details. Comment out to disable authentication
-      #- /cwa/config/path/app.db:/auth/app.db:ro

docs/configuration.md

@@ -0,0 +1,125 @@
+# Directory and Volume Setup
+
+This guide explains how to configure directories and Docker volumes for Shelfmark. It focuses on the difference between the destination folder and your download client paths, and how to make those paths line up inside containers.
+
+## Conceptual Overview
+
+```
+DIRECT DOWNLOADS
+
+Shelfmark downloads directly -> destination
+
+TORRENT / USENET
+
+Prowlarr -> Download client saves to <client path>
+         -> Shelfmark reads from <client path>
+         -> Shelfmark processes to destination
+```
+
+Key point: For torrent and usenet downloads, Shelfmark must see the same file path that your download client reports. The container path must match in both containers.
+
+## Direct Download Setup
+
+Direct downloads do not use an external download client. A simple two-folder setup is enough.
+
+Required volumes:
+
+| Container path | Purpose | Notes |
+| --- | --- | --- |
+| `/config` | Settings, database, cover cache | Configurable via `CONFIG_DIR` |
+| `/books` | Destination folder for completed files | Configurable via `INGEST_DIR` and Settings -> Downloads -> Destination |
+
+Example `docker-compose`:
+
+```yaml
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    volumes:
+      - /path/to/config:/config
+      - /path/to/books:/books
+```
+
+Notes:
+- Point `/books` to your library ingest folder (Calibre-Web, Booklore, Audiobookshelf, etc) for automatic import.
+- If you set Books Output Mode to Booklore (API), books are uploaded via API instead of written to `/books`. Audiobooks still use a destination folder.
+- Ensure `PUID`/`PGID` (or legacy `UID`/`GID`) match the owner of the host directories to avoid permission errors.
+
+## Torrent / Usenet Setup
+
+For torrents and usenet, your download client reports a path (for example `/data/torrents/books/MyBook.epub`). Shelfmark must be able to read that exact path inside its own container.
+
+Required volumes:
+
+| Container path | Purpose | Notes |
+| --- | --- | --- |
+| `/config` | Settings, database, cover cache | Configurable via `CONFIG_DIR` |
+| `/books` | Destination folder for processed files | Configurable via `INGEST_DIR` |
+| `<client path>` | Download client path | Must match the download client container path exactly |
+
+Side-by-side example with qBittorrent:
+
+```yaml
+services:
+  shelfmark:
+    volumes:
+      - /path/to/config:/config
+      - /path/to/books:/books
+      - /path/to/downloads:/data/torrents  # Must match client
+
+  qbittorrent:
+    volumes:
+      - /path/to/downloads:/data/torrents  # Same container path
+```
+
+Host paths can be anything. The container path (for example `/data/torrents`) must be identical in both containers.
+
+### Remote Path Mappings
+
+If paths cannot match (different machines or a fixed setup), use Remote Path Mappings.
+
+Where to configure:
+- Settings -> Advanced -> Remote Path Mappings
+
+Example:
+- Client reports `/data/torrents/books/...`
+- Shelfmark can see the same files at `/downloads/books/...`
+- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
+
+## File Processing Options
+
+### Transfer Method (Torrent / Usenet Only)
+
+Available methods:
+- Copy (default). Works everywhere.
+- Hardlink. Preserves seeding without duplicating files.
+
+Hardlink requirements and behavior:
+- Source and destination must be on the same filesystem.
+- If hardlinking is enabled but not possible, Shelfmark falls back to copying.
+- Archive extraction is disabled while hardlinking is enabled.
+- Do not use hardlinking if your destination is a library ingest folder.
+
+### File Organization
+
+Shelfmark supports three organization modes for the destination:
+- None. Keep original filenames from the source.
+- Rename Only. Rename files using a template.
+- Rename and Organize. Create folders and rename using templates. Do not use with ingest folders.
+
+Configure templates in Settings -> Downloads. Template syntax details are documented separately.
+
+## Common Mistakes
+
+- "Download failed - file not found": Path mismatch between Shelfmark and the download client. Ensure container paths match or use Remote Path Mappings.
+- "Permission denied": `PUID`/`PGID` do not match the host directories. Ensure Shelfmark can read the client path and write to the destination.
+- "Hardlinks not working" or "Files being copied instead": Source and destination are on different filesystems. Move the destination or accept copy fallback.
+- "Downloads work but library does not see them": Destination does not point to the library ingest folder. Check Settings -> Downloads -> Destination.
+- CIFS/SMB shares: Use the `nobrl` mount option to avoid database lock errors. Example: `//server/share /mnt/share cifs nobrl,... 0 0`
+
+## Related Documentation
+
+- Environment Variables Reference: `docs/environment-variables.md`
+- Custom Scripts: `docs/custom-scripts.md`
+- Installation: `docs/installation.md`
+- Troubleshooting: `docs/troubleshooting.md`

docs/custom-scripts.md

@@ -0,0 +1,184 @@
+# Custom Scripts
+
+Shelfmark can run an executable you provide after a download task completes successfully. The script runs after the selected output has finished (for example: transfer to the folder destination, or upload to Booklore).
+
+
+## Quick Start (Recommended)
+
+1. Put your script on the machine that runs Shelfmark.
+1. Make it executable.
+1. Set it in Shelfmark (Settings -> Advanced -> Custom Script Path).
+
+Example:
+
+```bash
+chmod +x /path/to/your/scripts/post_process.sh
+```
+
+### Docker Users
+
+If you run Shelfmark in Docker, the script must exist inside the container. The easiest way is to mount a folder of scripts, then point Shelfmark at the container path in the UI.
+
+```yaml
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    volumes:
+      - /path/to/your/scripts:/scripts:ro
+```
+
+Then set:
+
+- Settings -> Advanced -> Custom Script Path: `/scripts/post_process.sh`
+
+<details>
+<summary>Docker Compose: Configure Via Environment Variables (Optional)</summary>
+
+```yaml
+services:
+  shelfmark:
+    environment:
+      - CUSTOM_SCRIPT=/scripts/post_process.sh
+      - CUSTOM_SCRIPT_PATH_MODE=absolute
+      - CUSTOM_SCRIPT_JSON_PAYLOAD=true
+```
+
+</details>
+
+## Script Behaviour
+
+When enabled, Shelfmark runs your script once per successful task:
+
+```bash
+<custom_script_path> "<target_path>"
+```
+
+- `$1` is always set to the target path.
+- If **Custom Script JSON Payload** is enabled, Shelfmark writes a JSON document to stdin (UTF-8).
+- If JSON payload is disabled, stdin is empty (EOF).
+- Timeout: 300 seconds (5 minutes)
+- Exit code: `0` = success; anything else = the task is marked as **Error**
+- Concurrency: downloads can run in parallel, so your script may be invoked concurrently for different tasks.
+- Runtime: the script runs inside the Shelfmark container (if you use Docker) under the same user as Shelfmark.
+
+## The Target Path (`$1`)
+
+Shelfmark chooses a "best single path" for the task:
+
+- If the output produced exactly one local file: that file path.
+- If the output produced multiple local files: a directory path (the common parent directory of those files).
+
+What the target path refers to depends on the output mode:
+
+- Folder output (`output.mode=folder`, `phase=post_transfer`): the final imported file or folder inside your destination.
+- Booklore output (`output.mode=booklore`, `phase=post_upload`): the local file or folder that was uploaded (the destination is remote).
+
+By default, `$1` is an absolute path inside the Shelfmark container (or on your host, if you are not using Docker).
+
+## JSON Payload (stdin)
+
+Configure in: Settings -> Advanced -> Custom Script JSON Payload
+
+When enabled, Shelfmark sends a versioned JSON payload to your script via stdin (and still passes `$1`). This is the recommended way to write robust scripts, especially for multi-file imports (audiobooks) and output-specific context (like Booklore).
+
+- The JSON payload always includes absolute paths in `paths.*`, even if you set Custom Script Path Mode to `relative` for `$1`.
+- `output.mode` tells you which output ran.
+- `output.details` is output-specific. For Booklore output, `output.details.booklore` includes connection details such as `base_url`, `library_id`, and `path_id`.
+- `phase` indicates when the script is running. Current values: `post_transfer` (folder output), `post_upload` (Booklore output).
+- `transfer` is only included for outputs that do a local transfer (for example the folder output).
+
+If JSON payload is disabled, stdin is empty (EOF). Don't `cat` stdin unless you've enabled the payload.
+
+Example payload shape:
+
+```json
+{
+  "version": 1,
+  "phase": "post_transfer",
+  "task": {
+    "task_id": "abc123",
+    "source": "direct",
+    "title": "Foundation",
+    "author": "Isaac Asimov"
+  },
+  "output": {
+    "mode": "folder",
+    "organization_mode": "organize"
+  },
+  "paths": {
+    "destination": "/data/library/books",
+    "target": "/data/library/books/Isaac Asimov/Foundation/Foundation.epub",
+    "final_paths": [
+      "/data/library/books/Isaac Asimov/Foundation/Foundation.epub"
+    ]
+  },
+  "transfer": {
+    "op_counts": {"copy": 1, "move": 0, "hardlink": 0},
+    "use_hardlink": false,
+    "is_torrent": false,
+    "preserve_source": false
+  }
+}
+```
+
+Example (bash + jq) (JSON payload must be enabled):
+
+```bash
+payload="$(cat)"
+mode="$(echo "$payload" | jq -r '.output.mode')"
+title="$(echo "$payload" | jq -r '.task.title')"
+final_paths="$(echo "$payload" | jq -r '.paths.final_paths[]')"
+echo "mode=$mode title=$title" >&2
+echo "$final_paths" >&2
+```
+
+Example (Python) (works whether JSON payload is enabled or not):
+
+```python
+#!/usr/bin/env python3
+import json
+import sys
+
+target = sys.argv[1]
+raw = sys.stdin.read()
+payload = json.loads(raw) if raw.strip() else None
+
+print(f"target={target}", file=sys.stderr)
+if payload:
+    print(f"mode={payload['output']['mode']} phase={payload['phase']}", file=sys.stderr)
+```
+
+<details>
+<summary>Advanced Options</summary>
+
+### Absolute vs Relative Target Paths
+
+Configure in: Settings -> Advanced -> Custom Script Path Mode
+
+This setting controls what gets passed as `$1`:
+
+- `absolute` (default): pass an absolute path.
+- `relative`: pass a path relative to the output's "destination root", and run the script with `$PWD` set to that root.
+
+For folder output, the destination root is your configured destination folder. For Booklore output, it's the local upload folder.
+
+Example (folder destination is `/data/library/books`, and the imported file ended up in `Isaac Asimov/Foundation/Foundation.epub`):
+
+```bash
+# Absolute mode:
+$PWD is unchanged
+$1 = /data/library/books/Isaac Asimov/Foundation/Foundation.epub
+
+# Relative mode:
+$PWD = /data/library/books
+$1 = Isaac Asimov/Foundation/Foundation.epub
+```
+
+Note: if the target is the destination folder itself, `relative` mode may pass `.`.
+
+</details>
+
+## Notes And Caveats
+
+- **Hardlinks and torrents:** if you use hardlinking to keep seeding, avoid scripts that modify file contents, since hardlinked files share data with the seeding copy.
+- **Booklore output mode:** scripts run after upload. `$1` will point at the local uploaded file (or staging folder).

docs/dev/index.md

@@ -0,0 +1,3 @@
+# Developer Documentation
+
+TODO

docs/dev/plugin-settings.md

@@ -0,0 +1,654 @@
+# Plugin Settings Integration Guide
+
+This guide explains how to add configuration settings to plugins (Metadata Providers and Release Sources) so they appear in the Settings UI.
+
+## Overview
+
+The settings system uses a decorator-based registration pattern. Plugins register their settings when their module is imported, and the frontend dynamically renders the appropriate UI based on the schema provided by the backend.
+
+**Key features:**
+- Settings are defined in Python and automatically rendered in the React frontend
+- Values persist across container restarts via JSON config files
+- Changes take effect immediately without restart (unless marked otherwise)
+
+## Quick Start
+
+Add settings to your plugin in 3 steps:
+
+```python
+from shelfmark.core.settings_registry import (
+    register_settings,
+    TextField,
+    PasswordField,
+    ActionButton,
+)
+
+@register_settings(
+    name="my_plugin",           # Unique identifier
+    display_name="My Plugin",   # Shown in sidebar
+    icon="wrench",              # Icon name
+    order=100,                  # Sort order (lower = higher in list)
+    group="metadata_providers"  # Optional: group in sidebar
+)
+def my_plugin_settings():
+    return [
+        PasswordField(
+            key="MY_PLUGIN_API_KEY",
+            label="API Key",
+            description="Your API key from the provider",
+            required=True,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+def _test_connection():
+    # Perform connection test
+    return {"success": True, "message": "Connected successfully!"}
+```
+
+## Available Field Types
+
+### TextField
+
+Single-line text input for strings.
+
+```python
+TextField(
+    key="MY_SETTING",           # Config key
+    label="Setting Name",       # Display label
+    description="Help text",    # Optional description below field
+    default="",                 # Default value
+    placeholder="Enter value",  # Placeholder text
+    max_length=100,             # Optional max characters
+    required=False,             # Is this field required?
+    requires_restart=False,     # Does changing this need a restart?
+    show_when=None,             # Conditional visibility (see below)
+    disabled_when=None,         # Conditional disable (see below)
+)
+```
+
+### PasswordField
+
+Masked input for sensitive values (API keys, passwords). Values are never echoed back to the frontend.
+
+```python
+PasswordField(
+    key="API_KEY",
+    label="API Key",
+    description="Your secret API key",
+    placeholder="sk-...",
+    required=True,
+)
+```
+
+### NumberField
+
+Numeric input with optional min/max constraints.
+
+```python
+NumberField(
+    key="TIMEOUT",
+    label="Timeout (seconds)",
+    description="Connection timeout in seconds",
+    default=30,
+    min_value=5,
+    max_value=300,
+    step=1,                     # Increment step
+    required=False,
+)
+```
+
+### CheckboxField
+
+Toggle switch for boolean values.
+
+```python
+CheckboxField(
+    key="ENABLE_FEATURE",
+    label="Enable Feature",
+    description="Turn this feature on or off",
+    default=False,
+)
+```
+
+### SelectField
+
+Dropdown for single-choice selection.
+
+```python
+SelectField(
+    key="LOG_LEVEL",
+    label="Log Level",
+    description="Logging verbosity",
+    default="info",
+    options=[
+        {"value": "debug", "label": "Debug"},
+        {"value": "info", "label": "Info"},
+        {"value": "warning", "label": "Warning"},
+        {"value": "error", "label": "Error"},
+    ],
+)
+```
+
+### MultiSelectField
+
+Multi-choice selection from a list of options.
+
+```python
+MultiSelectField(
+    key="SUPPORTED_FORMATS",
+    label="Supported Formats",
+    description="Select which formats to support",
+    default=["epub", "mobi"],
+    options=[
+        {"value": "epub", "label": "EPUB"},
+        {"value": "mobi", "label": "MOBI"},
+        {"value": "pdf", "label": "PDF"},
+        {"value": "azw3", "label": "AZW3"},
+    ],
+)
+```
+
+### ActionButton
+
+Button that executes a callback function. Does not store a value.
+
+```python
+ActionButton(
+    key="test_connection",      # Unique key for the action
+    label="Test Connection",    # Button text
+    description="Test the API connection",
+    style="primary",            # "default", "primary", or "danger"
+    callback=my_callback_fn,    # Function to execute
+)
+
+def my_callback_fn():
+    """Callback must return dict with 'success' and 'message' keys."""
+    try:
+        # Perform action
+        return {"success": True, "message": "Connection successful!"}
+    except Exception as e:
+        return {"success": False, "message": f"Failed: {str(e)}"}
+```
+
+### HeadingField
+
+Display-only section heading with optional link. Does not store a value.
+
+```python
+HeadingField(
+    key="section_heading",      # Unique key
+    title="Configuration",      # Heading text
+    description="Configure the plugin settings below",
+    link_url="https://example.com/docs",  # Optional link
+    link_text="View Documentation",       # Link text
+)
+```
+
+### CustomComponentField
+
+Render a frontend-registered custom settings component while still using the
+decorator-based schema.
+
+```python
+from shelfmark.core.settings_registry import CustomComponentField
+
+CustomComponentField(
+    key="request_policy_editor",
+    component="request_policy_grid",  # frontend registry key
+    label="Request Policy Rules",
+    description="Custom editor for policy defaults and matrix rules.",
+    value_fields=[
+        SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
+        SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
+        TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
+    ],
+    wrap_in_field_wrapper=True,  # use standard FieldWrapper label/description layout
+)
+```
+
+When `value_fields` is provided, those backing fields are included in
+serialization/save/validation automatically and are hidden from the default renderer.
+
+## Common Field Properties
+
+All field types support these common properties:
+
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `key` | `str` | Required | Unique identifier for this setting |
+| `label` | `str` | Required | Display label in the UI |
+| `description` | `str` | `""` | Help text shown below the field |
+| `default` | `Any` | `None` | Default value if not set |
+| `required` | `bool` | `False` | Whether the field must have a value |
+| `disabled` | `bool` | `False` | Disable the field (greyed out) |
+| `disabled_reason` | `str` | `""` | Explanation shown when disabled |
+| `requires_restart` | `bool` | `False` | Whether changes require container restart |
+| `show_when` | `dict` | `None` | Conditional visibility (see below) |
+| `disabled_when` | `dict` | `None` | Conditional disable (see below) |
+| `hidden_in_ui` | `bool` | `False` | Hide from default renderer but keep in schema/save path |
+
+## Conditional Visibility
+
+Fields can be shown/hidden based on other field values using `show_when`:
+
+```python
+# Only show DNS servers field when custom DNS is selected
+TextField(
+    key="CUSTOM_DNS_SERVERS",
+    label="DNS Servers",
+    description="Comma-separated DNS server IPs",
+    show_when={"field": "DNS_PROVIDER", "value": "manual"},
+)
+```
+
+The field will only be visible when the referenced field has the specified value.
+
+## Conditional Disable
+
+Fields can be enabled/disabled based on other field values using `disabled_when`:
+
+```python
+# Disable timeout field when feature is disabled
+NumberField(
+    key="FEATURE_TIMEOUT",
+    label="Timeout (seconds)",
+    description="Request timeout",
+    default=30,
+    disabled_when={
+        "field": "FEATURE_ENABLED",
+        "value": False,
+        "reason": "Enable the feature first"
+    },
+)
+```
+
+The field will be greyed out with the specified reason when the condition is met.
+
+## Settings Groups
+
+Register a group to organize related settings tabs in the sidebar:
+
+```python
+from shelfmark.core.settings_registry import register_group
+
+# Register a group (do this once, usually in a central config file)
+register_group(
+    name="my_group",
+    display_name="My Group",
+    icon="folder",
+    order=50,
+)
+
+# Then register settings to the group
+@register_settings(
+    name="plugin_a",
+    display_name="Plugin A",
+    icon="puzzle",
+    order=51,
+    group="my_group",  # Assigns to the group
+)
+def plugin_a_settings():
+    return [...]
+```
+
+**Existing groups:**
+- `direct_download` (order=20): For download-related settings
+- `metadata_providers` (order=50): For metadata provider plugins
+
+## Value Resolution Priority
+
+Settings values are resolved in this order (highest priority first):
+
+1. **Config File** - Stored in `CONFIG_DIR/plugins/<tab_name>.json`
+2. **Field Default** - Value specified in the field definition
+
+The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdirectory.
+
+## Reading Setting Values
+
+Use the `config` singleton to read setting values in your plugin code:
+
+```python
+from shelfmark.core.config import config
+
+# Get a setting value with default fallback
+api_key = config.get("MY_PLUGIN_API_KEY", "")
+timeout = config.get("MY_PLUGIN_TIMEOUT", 30)
+
+# Or access as attributes (raises AttributeError if not found)
+api_key = config.MY_PLUGIN_API_KEY
+
+# Check all cached settings
+all_settings = config.get_all()
+```
+
+The config singleton:
+- Automatically resolves values from config files with field defaults as fallback
+- Caches values for performance
+- Refreshes automatically when settings are updated via the UI
+
+## Complete Example: Metadata Provider
+
+Here's a complete example for a metadata provider plugin:
+
+```python
+# shelfmark/metadata_providers/my_provider.py
+
+from shelfmark.metadata_providers.base import (
+    MetadataProvider,
+    register_provider,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_connection():
+    """Test API connection callback."""
+    api_key = config.get("MY_PROVIDER_API_KEY", "")
+    if not api_key:
+        return {"success": False, "message": "API key not configured"}
+
+    try:
+        # Perform actual connection test
+        # response = requests.get(...)
+        return {"success": True, "message": "Connected to My Provider API"}
+    except Exception as e:
+        return {"success": False, "message": f"Connection failed: {str(e)}"}
+
+
+@register_settings(
+    name="my_provider",
+    display_name="My Provider",
+    icon="book",
+    order=53,
+    group="metadata_providers",
+)
+def my_provider_settings():
+    """Define settings for this metadata provider."""
+    return [
+        HeadingField(
+            key="my_provider_heading",
+            title="My Provider",
+            description="A metadata provider for book information",
+            link_url="https://myprovider.com",
+            link_text="Visit My Provider",
+        ),
+        PasswordField(
+            key="MY_PROVIDER_API_KEY",
+            label="API Key",
+            description="Your My Provider API key",
+            placeholder="Enter your API key",
+            required=True,
+        ),
+        CheckboxField(
+            key="MY_PROVIDER_INCLUDE_COVERS",
+            label="Include Cover Images",
+            description="Fetch cover images when searching",
+            default=True,
+        ),
+        TextField(
+            key="MY_PROVIDER_BASE_URL",
+            label="API Base URL",
+            description="Override the default API endpoint",
+            default="https://api.myprovider.com/v1",
+            required=False,
+        ),
+        ActionButton(
+            key="test_connection",
+            label="Test Connection",
+            description="Verify your API key works",
+            style="primary",
+            callback=_test_connection,
+        ),
+    ]
+
+
+@register_provider("my_provider")
+class MyProvider(MetadataProvider):
+    """My Provider metadata implementation."""
+
+    name = "my_provider"
+    display_name = "My Provider"
+    requires_auth = True
+
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
+        self.base_url = config.get(
+            "MY_PROVIDER_BASE_URL",
+            "https://api.myprovider.com/v1"
+        )
+
+    def is_available(self) -> bool:
+        return bool(self.api_key)
+
+    def search(self, query: str):
+        # Implementation...
+        pass
+
+    def get_book(self, book_id: str):
+        # Implementation...
+        pass
+```
+
+## Complete Example: Release Source
+
+Here's a complete example for a release source plugin:
+
+```python
+# shelfmark/release_sources/my_source.py
+
+from shelfmark.release_sources.base import (
+    ReleaseSource,
+    DownloadHandler,
+    register_source,
+    register_handler,
+)
+from shelfmark.core.settings_registry import (
+    register_settings,
+    HeadingField,
+    TextField,
+    NumberField,
+    CheckboxField,
+    SelectField,
+    ActionButton,
+)
+from shelfmark.core.config import config
+
+
+def _test_source():
+    """Test source availability callback."""
+    base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+    try:
+        # Test connectivity
+        return {"success": True, "message": f"Source available at {base_url}"}
+    except Exception as e:
+        return {"success": False, "message": f"Source unavailable: {str(e)}"}
+
+
+@register_settings(
+    name="my_source",
+    display_name="My Source",
+    icon="download",
+    order=25,
+    group="direct_download",
+)
+def my_source_settings():
+    """Define settings for this release source."""
+    return [
+        HeadingField(
+            key="my_source_heading",
+            title="My Source Configuration",
+            description="Configure the My Source download provider",
+        ),
+        CheckboxField(
+            key="MY_SOURCE_ENABLED",
+            label="Enable My Source",
+            description="Include My Source in download fallback chain",
+            default=True,
+        ),
+        TextField(
+            key="MY_SOURCE_URL",
+            label="Source URL",
+            description="Base URL for the source",
+            default="https://mysource.com",
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        NumberField(
+            key="MY_SOURCE_TIMEOUT",
+            label="Timeout (seconds)",
+            description="Request timeout",
+            default=30,
+            min_value=10,
+            max_value=120,
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        SelectField(
+            key="MY_SOURCE_PRIORITY",
+            label="Priority",
+            description="Where in the fallback chain to try this source",
+            default="normal",
+            options=[
+                {"value": "high", "label": "High (try first)"},
+                {"value": "normal", "label": "Normal"},
+                {"value": "low", "label": "Low (try last)"},
+            ],
+            show_when={"field": "MY_SOURCE_ENABLED", "value": True},
+        ),
+        ActionButton(
+            key="test_source",
+            label="Test Source",
+            description="Check if the source is accessible",
+            style="primary",
+            callback=_test_source,
+        ),
+    ]
+
+
+@register_source("my_source")
+class MySource(ReleaseSource):
+    """My Source release source implementation."""
+
+    name = "my_source"
+    display_name = "My Source"
+
+    def __init__(self):
+        self.enabled = config.get("MY_SOURCE_ENABLED", True)
+        self.base_url = config.get("MY_SOURCE_URL", "https://mysource.com")
+        self.timeout = config.get("MY_SOURCE_TIMEOUT", 30)
+
+    def is_available(self) -> bool:
+        return self.enabled
+
+    def search(self, book):
+        # Implementation...
+        pass
+
+
+@register_handler("my_source")
+class MySourceHandler(DownloadHandler):
+    """Handler for downloading from My Source."""
+
+    name = "my_source"
+
+    def download(self, release, output_path):
+        # Implementation...
+        pass
+```
+
+## Best Practices
+
+1. **Use descriptive keys**: Keys should be uppercase and prefixed with your plugin name (e.g., `MY_PLUGIN_API_KEY`)
+
+2. **Provide helpful descriptions**: Include enough detail in descriptions to help users understand what each setting does
+
+3. **Set sensible defaults**: Users should be able to get started without configuring everything
+
+4. **Use conditional visibility**: Hide advanced options behind enabling checkboxes to reduce UI clutter
+
+5. **Include a test button**: ActionButtons that test connections help users verify their configuration
+
+6. **Mark restart-required settings**: Use `requires_restart=True` for settings that can't be applied live
+
+7. **Group related settings**: Use HeadingField to visually separate sections, and put plugins in appropriate groups
+
+8. **Handle missing values gracefully**: Always provide fallbacks when reading settings in your code
+
+## API Reference
+
+### Backend Routes
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/api/settings` | Get all settings tabs, groups, and values |
+| GET | `/api/settings/<tab_name>` | Get a specific settings tab |
+| PUT | `/api/settings/<tab_name>` | Update settings for a tab |
+| POST | `/api/settings/<tab_name>/action/<action_key>` | Execute an action button callback |
+
+### Response Format
+
+**GET /api/settings**
+```json
+{
+  "groups": [
+    {"name": "direct_download", "displayName": "Direct Download", "icon": "download", "order": 20}
+  ],
+  "tabs": [
+    {
+      "name": "my_plugin",
+      "displayName": "My Plugin",
+      "icon": "book",
+      "order": 53,
+      "group": "metadata_providers",
+      "fields": [
+        {
+          "type": "password",
+          "key": "MY_PLUGIN_API_KEY",
+          "label": "API Key",
+          "description": "Your API key",
+          "hasValue": true,
+          "value": "",
+          "required": true,
+          "disabled": false,
+          "requiresRestart": false
+        }
+      ]
+    }
+  ]
+}
+```
+
+**PUT /api/settings/<tab_name>**
+```json
+// Request
+{"MY_PLUGIN_API_KEY": "new-value", "MY_PLUGIN_TIMEOUT": 60}
+
+// Response
+{
+  "success": true,
+  "message": "Settings updated",
+  "updated": ["MY_PLUGIN_API_KEY", "MY_PLUGIN_TIMEOUT"],
+  "requiresRestart": false
+}
+```
+
+**POST /api/settings/<tab_name>/action/<action_key>**
+```json
+// Response
+{
+  "success": true,
+  "message": "Connection successful!"
+}
+```

docs/index.md

@@ -0,0 +1,3 @@
+# Shelfmark Documentation
+
+TODO

docs/installation.md

@@ -0,0 +1,3 @@
+# Installation
+
+TODO

docs/oidc.md

@@ -0,0 +1,50 @@
+# OpenID Connect (OIDC) Authentication
+
+## Callback URL
+
+```
+https://<your-shelfmark-domain>/api/auth/oidc/callback
+```
+
+With a subpath (`URL_BASE=/shelfmark/`):
+
+```
+https://<your-shelfmark-domain>/shelfmark/api/auth/oidc/callback
+```
+
+The callback URL is constructed from the incoming request, so your reverse proxy must forward `X-Forwarded-Proto` and `X-Forwarded-Host` correctly. PKCE (S256) is used automatically.
+
+## Settings
+
+Configure in **Settings → Security → Authentication Method → OIDC**.
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| Discovery URL | `/.well-known/openid-configuration` endpoint | — |
+| Client ID | OAuth2 client ID | — |
+| Client Secret | OAuth2 client secret | — |
+| Scopes | Scopes to request. The group claim is added automatically when admin group authorization is enabled | `openid email profile` |
+| Group Claim Name | Claim containing user groups | `groups` |
+| Admin Group Name | Group granted admin access. Leave empty for database-only roles | — |
+| Use Admin Group for Authorization | Toggle group-based admin detection | `true` |
+| Auto-Provision Users | Create accounts on first login | `true` |
+| Login Button Label | Custom text for the sign-in button | — |
+
+Use **Test Connection** to verify discovery and client configuration before attempting login.
+
+## Environment Variables
+
+These optional environment variables control login page behavior when OIDC is enabled.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `HIDE_LOCAL_AUTH` | Hide the username/password login option, so only the OIDC button is shown | `false` |
+| `OIDC_AUTO_REDIRECT` | Automatically redirect to the OIDC provider instead of showing the login page | `false` |
+
+If both are enabled, users are redirected straight to the OIDC provider. On failure they return to the login page with an error message but no password fallback.
+
+## Troubleshooting
+
+- **Issuer validation failed** — The issuer in the token doesn't match the discovery document. Check your provider's external URL / issuer configuration.
+- **Callback URL mismatch** — Reverse proxy isn't forwarding `X-Forwarded-Proto` or `X-Forwarded-Host`, so the constructed callback URL doesn't match what's registered in the provider.
+- **Account not found** — Auto-provision is disabled and the user hasn't been pre-created by an admin.

docs/reverse-proxy.md

@@ -0,0 +1,147 @@
+# Reverse Proxy & Subpath Hosting
+
+Shelfmark can run behind a reverse proxy at the root path (recommended) or under a subpath like `/shelfmark`.
+
+## Root path setup (Recommended)
+
+If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`), leave `URL_BASE` empty. This is the simplest option and avoids extra subpath configuration.
+
+```nginx
+server {
+    listen 443 ssl;
+    server_name shelfmark.example.com;
+
+    location / {
+        proxy_pass http://shelfmark:8084;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
+```
+
+## Subpath setup
+
+Running Shelfmark under a subpath like `/shelfmark` is supported without extra rewrite rules.
+
+### 1. Set the base path in Shelfmark
+
+- **UI**: Settings → Advanced → Base Path → `/shelfmark/`
+- **Environment variable**: `URL_BASE=/shelfmark/`
+
+### 2. Configure your reverse proxy
+
+All Shelfmark paths (UI, API, assets, Socket.IO) are served under the base path. A single location block is enough.
+
+---
+
+### Without Authentication Proxy
+
+**Complete Nginx configuration for subpath deployment:**
+
+```nginx
+location /shelfmark/ {
+    proxy_pass http://shelfmark:8084/shelfmark/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+    proxy_send_timeout 86400;
+    proxy_buffering off;
+}
+```
+
+---
+
+### With Authentication Proxy (Authelia, Authentik, etc.)
+
+Shelfmark supports Proxy Authentication. When enabled, Shelfmark trusts the authenticated user from headers set by your auth proxy.
+
+#### Shelfmark Settings
+
+Configure in Settings → Security:
+
+| Setting | Value |
+|---------|-------|
+| Authentication Method | Proxy Authentication |
+| Proxy Auth User Header | `Remote-User` |
+| Proxy Auth Logout URL | `https://auth.example.com/logout` |
+| Proxy Auth Admin Group Header | `Remote-Groups` |
+| Proxy Auth Admin Group Name | `admins` (or your admin group) |
+
+#### Nginx Configuration with Authelia
+
+This example uses Authelia snippets. Adapt for your auth proxy.
+
+**Authelia auth request snippet** (`/etc/nginx/snippets/authelia-authrequest.conf`):
+
+```nginx
+location /authelia {
+    internal;
+    proxy_pass http://authelia:9091/api/authz/auth-request;
+    proxy_pass_request_body off;
+    proxy_set_header Content-Length "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
+    proxy_set_header X-Original-Method $request_method;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+}
+```
+
+**Authelia location snippet** (`/etc/nginx/snippets/authelia-location.conf`):
+
+```nginx
+auth_request /authelia;
+auth_request_set $target_url $scheme://$http_host$request_uri;
+auth_request_set $user $upstream_http_remote_user;
+auth_request_set $groups $upstream_http_remote_groups;
+auth_request_set $name $upstream_http_remote_name;
+auth_request_set $email $upstream_http_remote_email;
+proxy_set_header Remote-User $user;
+proxy_set_header Remote-Groups $groups;
+proxy_set_header Remote-Name $name;
+proxy_set_header Remote-Email $email;
+error_page 401 =302 https://auth.example.com/?rd=$target_url;
+```
+
+**Complete Nginx configuration with Authelia:**
+
+```nginx
+# Include Authelia auth endpoint in your server block
+include /etc/nginx/snippets/authelia-authrequest.conf;
+
+# Main shelfmark location
+location /shelfmark/ {
+    include /etc/nginx/snippets/authelia-location.conf;
+
+    proxy_pass http://shelfmark:8084/shelfmark/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+    proxy_send_timeout 86400;
+    proxy_buffering off;
+}
+```
+
+---
+
+## Health checks
+
+Health checks work at `/shelfmark/api/health` when using a subpath configuration.

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+# Troubleshooting
+
+TODO

docs/url-search-parameters.md

@@ -0,0 +1,82 @@
+# URL Search Parameters
+
+You can trigger searches directly via URL by adding query parameters. This enables bookmarking searches and sharing links.
+
+## Basic Usage
+
+```
+http://your-server:8084/?q=harry+potter
+```
+
+## Supported Parameters
+
+| Parameter | Description | Example |
+|-----------|-------------|---------|
+| `q` or `query` | Main search query | `/?q=dune` |
+| `author` | Filter by author name | `/?author=frank+herbert` |
+| `title` | Filter by book title | `/?title=foundation` |
+| `isbn` | Filter by ISBN | `/?isbn=978-0747532699` |
+| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` |
+| `format` | Filter by file format | `/?format=epub` |
+| `content` | Filter by content type | `/?content=fiction` |
+| `content_type` | Select media type (`ebook` or `audiobook`) in Universal mode only | `/?q=dune&content_type=audiobook` |
+| `sort` | Sort order for results | `/?sort=newest` |
+
+## Multiple Values
+
+Some parameters support multiple values by repeating the parameter:
+
+```
+/?lang=en&lang=de&lang=fr
+/?format=epub&format=mobi&format=azw3
+```
+
+## Examples
+
+**Simple search:**
+```
+/?q=lord+of+the+rings
+```
+
+**Search with author filter:**
+```
+/?q=dune&author=frank+herbert
+```
+
+**Search with format and language:**
+```
+/?q=harry+potter&format=epub&lang=en
+```
+
+**Author search with multiple formats:**
+```
+/?author=stephen+king&format=epub&format=mobi
+```
+
+**Search with sort order:**
+```
+/?q=science+fiction&sort=newest
+```
+
+**Universal search as audiobook:**
+```
+/?q=dune&content_type=audiobook
+```
+
+## Search Mode Behavior
+
+### Direct Download Mode (default)
+
+All parameters are used to filter results from the direct download source.
+`content_type` is ignored in Direct mode.
+
+### Universal Mode
+
+`q`, `sort`, and `content_type` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities.
+
+## Notes
+
+- URL parameters are read once on page load
+- The URL is not updated when you perform searches manually
+- Spaces should be encoded as `+` or `%20`
+- Invalid or unknown parameters are silently ignored

docs/users-and-requests.md

@@ -0,0 +1,98 @@
+# Users & Requests
+
+Configure in **Settings → Users & Requests**.
+
+## Authentication Methods
+
+Shelfmark supports four authentication methods, configured in **Settings → Security**.
+
+### Local
+
+You create user accounts directly in Shelfmark with a username and password. At least one local admin account must exist before this mode can be enabled.
+
+### Proxy Authentication
+
+Your reverse proxy handles authentication and passes the username to Shelfmark via a header (e.g. `Remote-User`). Accounts are created automatically on first sign-in. If a local user with the same username already exists, the proxy identity will be linked to that account rather than creating a duplicate. Admin status can optionally be derived from a groups header.
+
+### OIDC (OpenID Connect)
+
+Users sign in through your identity provider. Accounts are created automatically on first login (unless auto-provisioning is disabled, in which case you need to pre-create them). If a local user with a matching verified email already exists, the OIDC identity will be linked to that account on first sign-in. Admin status can optionally be derived from a group claim.
+
+A local admin account is required as a fallback. See [OIDC](oidc.md) for provider setup.
+
+### Calibre-Web Database
+
+User accounts are synced from your Calibre-Web `app.db`. If a local user with a matching email already exists, the CWA identity will be linked to that account. Roles are kept in sync with CWA. Users removed from CWA are cleaned up on the next sync.
+
+Requires mounting your Calibre-Web `app.db` to `/auth/app.db`.
+
+## Per-User Settings
+
+Admins can configure per-user settings by editing a user in the user management panel. Non-admin users can also edit their own settings through **My Account** (accessible from the user menu). Admins control which sections are visible in My Account via the **Visible Self-Settings Sections** option.
+
+There are three categories of per-user settings:
+
+### Delivery Preferences
+
+Override where a user's downloads are sent. Options depend on the global output mode configuration:
+
+- **Output mode** — Folder, Email (SMTP), or BookLore (API)
+- **Destination** — A custom folder path for this user's ebook downloads
+- **Audiobook destination** — A custom folder path for audiobook downloads
+- **BookLore library/path** — Per-user BookLore target (when using BookLore output mode)
+- **Email recipient** — Per-user email address (when using Email output mode)
+
+### Notifications
+
+Users can configure personal notification routes, separate from the global notification settings. Each route targets a URL (e.g. an Apprise-compatible endpoint) and can be scoped to specific event types or all events.
+
+### Request Policy (admin-only)
+
+Admins can override the default ebook/audiobook modes and request rules for individual users. See [Per-User Overrides](#per-user-overrides) below.
+
+---
+
+## Requests
+
+The request system controls whether users can download directly or need admin approval first.
+
+### Policy Modes
+
+Each content type (ebook, audiobook) has a default mode that sets the baseline:
+
+| Mode | Behaviour |
+|------|-----------|
+| **Download** | Users download directly, no approval needed |
+| **Request Release** | Users pick a specific release, then submit it for admin approval |
+| **Request Book** | Users request the book itself — an admin picks the release and fulfils it |
+| **Blocked** | No downloads or requests allowed |
+
+### Settings
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| Enable Requests | Master toggle. When off, everyone downloads directly | Off |
+| Default Ebook Mode | Baseline mode for all ebook sources | Download |
+| Default Audiobook Mode | Baseline mode for all audiobook sources | Download |
+| Request Rules | Per-source overrides (see below) | None |
+| Max Pending Requests Per User | Open request limit per user | 20 |
+| Allow Notes on Requests | Let users attach a note when submitting | On |
+
+### Request Rules
+
+The rules matrix lets you override the mode for specific source + content type combinations. Rules can only be **equal to or more restrictive** than the content-type default — they cannot grant more access than the baseline.
+
+For example, if the default ebook mode is "Download", a rule can restrict a specific source to "Request Release" or "Blocked", but not the other way around. If no rule matches, the content-type default applies.
+
+### Per-User Overrides
+
+Admins can override the default ebook/audiobook modes and request rules for individual users. Per-user rules are overlaid on the global rules, not replacing them.
+
+### Request Lifecycle
+
+1. User submits a request (book or release level, depending on the resolved policy mode)
+2. Request appears in the admin request queue as **pending**
+3. Admin either **fulfils** (queues a download) or **rejects** the request
+4. For fulfilled requests, delivery state is tracked through the download pipeline
+5. If delivery fails, an admin can reopen the request to try a different release
+6. Users can cancel their own pending requests

downloader.py

@@ -1,131 +0,0 @@
-"""Network operations manager for the book downloader application."""
-
-import network
-network.init()
-import requests
-import time
-from io import BytesIO
-from typing import Optional
-from urllib.parse import urlparse
-from tqdm import tqdm
-
-from logger import setup_logger
-from config import PROXIES
-from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS
-if USE_CF_BYPASS:
-    import cloudflare_bypasser
-
-logger = setup_logger(__name__)
-
-
-def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False) -> str:
-    """Fetch HTML content from a URL with retry mechanism.
-    
-    Args:
-        url: Target URL
-        retry: Number of retry attempts
-        skip_404: Whether to skip 404 errors
-        
-    Returns:
-        str: HTML content if successful, None otherwise
-    """
-    response = None
-    try:
-        logger.debug(f"html_get_page: {url}, retry: {retry}, use_bypasser: {use_bypasser}")
-        if use_bypasser and USE_CF_BYPASS:
-            logger.info(f"GET Using Cloudflare Bypasser for: {url}")
-            response_html = cloudflare_bypasser.get(url)
-            logger.debug(f"Cloudflare Bypasser response length: {len(response_html)}")
-            if response_html.strip() != "":
-                return response_html
-            else:
-                raise requests.exceptions.RequestException("Failed to bypass Cloudflare")
-        else:
-            logger.info(f"GET: {url}")
-            response = requests.get(url, proxies=PROXIES)
-            response.raise_for_status()
-            logger.debug(f"Success getting: {url}")
-            time.sleep(1)
-        return str(response.text)
-        
-    except Exception as e:
-        if retry == 0:
-            logger.error_trace(f"Failed to fetch page: {url}, error: {e}")
-            return ""
-        
-        if use_bypasser and USE_CF_BYPASS:
-            logger.warning(f"Exception while using cloudflare bypass for URL: {url}")
-            logger.warning(f"Exception: {e}")
-            logger.warning(f"Response: {response}")
-        elif response is not None and response.status_code == 404:
-            logger.warning(f"404 error for URL: {url}")
-            return ""
-        elif response is not None and response.status_code == 403:
-            logger.warning(f"403 detected for URL: {url}. Should retry using cloudflare bypass.")
-            return html_get_page(url, retry - 1, True)
-            
-        sleep_time = DEFAULT_SLEEP * (MAX_RETRY - retry + 1)
-        logger.warning(
-            f"Retrying GET {url} in {sleep_time} seconds due to error: {e}"
-        )
-        time.sleep(sleep_time)
-        return html_get_page(url, retry - 1, use_bypasser)
-
-def download_url(link: str, size: str = "") -> Optional[BytesIO]:
-    """Download content from URL into a BytesIO buffer.
-    
-    Args:
-        link: URL to download from
-        
-    Returns:
-        BytesIO: Buffer containing downloaded content if successful
-    """
-    try:
-        logger.info(f"Downloading from: {link}")
-        response = requests.get(link, stream=True, proxies=PROXIES)
-        response.raise_for_status()
-
-        total_size : float = 0.0
-        try:
-            # we assume size is in MB
-            total_size = float(size.strip().replace(" ", "").replace(",", ".").upper()[:-2].strip()) * 1024 * 1024
-        except:
-            total_size = float(response.headers.get('content-length', 0))
-        
-        buffer = BytesIO()
-
-        # Initialize the progress bar with your guess
-        pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
-        for chunk in response.iter_content(chunk_size=1000):
-            buffer.write(chunk)
-            pbar.update(len(chunk))
-            
-        pbar.close()
-        if buffer.tell() * 0.1 < total_size * 0.9:
-            # Check the content of the buffer if its HTML or binary
-            if response.headers.get('content-type', '').startswith('text/html'):
-                logger.warn(f"Failed to download content for {link}. Found HTML content instead.")
-                return None
-        return buffer
-    except requests.exceptions.RequestException as e:
-        logger.error_trace(f"Failed to download from {link}: {e}")
-        return None
-
-def get_absolute_url(base_url: str, url: str) -> str:
-    """Get absolute URL from relative URL and base URL.
-    
-    Args:
-        base_url: Base URL
-        url: Relative URL
-    """
-    if url.strip() == "":
-        return ""
-    if url.strip("#") == "":
-        return ""
-    if url.startswith("http"):
-        return url
-    parsed_url = urlparse(url)
-    parsed_base = urlparse(base_url)
-    if parsed_url.netloc == "" or parsed_url.scheme == "":
-        parsed_url = parsed_url._replace(netloc=parsed_base.netloc, scheme=parsed_base.scheme)
-    return parsed_url.geturl()

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/cwa-book-downloader
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/cwa-bd_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+    # Keep the previous entrypoint log instead of deleting all history on boot.
+    [ -f "${LOG_FILE}.prev" ] && rm -f "${LOG_FILE}.prev"
+    [ -f "$LOG_FILE" ] && mv "$LOG_FILE" "${LOG_FILE}.prev"
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,14 +56,21 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
 echo "Build version: $BUILD_VERSION"
+echo "Release version: $RELEASE_VERSION"
 
 # Configure timezone
 if [ "$TZ" ]; then
@@ -27,40 +78,75 @@ if [ "$TZ" ]; then
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 fi
 
-# Set UID if not set
-if [ -z "$UID" ]; then
-    UID=1000
+# Determine user ID with proper precedence:
+# 1. PUID (LinuxServer.io standard - recommended)
+# 2. UID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+#
+# Note: $UID is a bash builtin that's always set. We use `printenv` to detect
+# if UID was explicitly set as an environment variable (e.g., via docker-compose).
+if [ -n "$PUID" ]; then
+    RUN_UID="$PUID"
+    echo "Using PUID=$RUN_UID"
+elif printenv UID >/dev/null 2>&1; then
+    RUN_UID="$(printenv UID)"
+    echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)"
+else
+    RUN_UID=1000
+    echo "Using default UID=$RUN_UID"
 fi
 
-# Set GID if not set
-if [ -z "$GID" ]; then
-    GID=100
+# Determine group ID with proper precedence:
+# 1. PGID (LinuxServer.io standard - recommended)
+# 2. GID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+if [ -n "$PGID" ]; then
+    RUN_GID="$PGID"
+    echo "Using PGID=$RUN_GID"
+elif [ -n "$GID" ]; then
+    RUN_GID="$GID"
+    echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)"
+else
+    RUN_GID=1000
+    echo "Using default GID=$RUN_GID"
 fi
 
-if ! getent group "$GID" >/dev/null; then
-    echo "Adding group $GID with name appuser"
-    groupadd -g "$GID" appuser
+if ! getent group "$RUN_GID" >/dev/null; then
+    echo "Adding group $RUN_GID with name appuser"
+    groupadd -g "$RUN_GID" appuser
 fi
 
 # Create user if it doesn't exist
-if ! id -u "$UID" >/dev/null 2>&1; then
-    echo "Adding user $UID with name appuser"
-    useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser
+if ! id -u "$RUN_UID" >/dev/null 2>&1; then
+    echo "Adding user $RUN_UID with name appuser"
+    useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser
 fi
 
 # Get username for the UID (whether we just created it or it existed)
-USERNAME=$(getent passwd "$UID" | cut -d: -f1)
-echo "Username for UID $UID is $USERNAME"
+USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1)
+echo "Username for UID $RUN_UID is $USERNAME"
 
 test_write() {
-    folder=$1
-    test_file=$folder/calibre-web-automated-book-downloader_TEST_WRITE
-    mkdir -p $folder
-    (
-        echo 0123456789_TEST | sudo -E -u "$USERNAME" HOME=/app tee $test_file > /dev/null
-    )
-    FILE_CONTENT=$(cat $test_file || echo "")
-    rm -f $test_file
+    local folder=$1
+    local test_file="$folder/shelfmark_TEST_WRITE"
+    local FILE_CONTENT
+    local result
+    local result_text
+
+    if ! mkdir -p "$folder"; then
+        echo "Failed to create directory for write test: $folder"
+        return 1
+    fi
+
+    if ! (
+        echo 0123456789_TEST | gosu "$USERNAME" env HOME=/app tee "$test_file" > /dev/null
+    ); then
+        echo "Failed to write test file in $folder as $USERNAME"
+        return 1
+    fi
+
+    FILE_CONTENT=$(cat "$test_file" 2>/dev/null || echo "")
+    rm -f "$test_file"
     [ "$FILE_CONTENT" = "0123456789_TEST" ]
     result=$?
     if [ $result -eq 0 ]; then
@@ -74,6 +160,7 @@ test_write() {
 
 make_writable() {
     folder=$1
+    did_full_chown=0
     set +e
     test_write $folder
     is_writable=$?
@@ -83,37 +170,118 @@ make_writable() {
     else
         echo "Folder $folder is not writable, changing ownership"
         change_ownership $folder
-        chmod g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        chmod -R g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        did_full_chown=1
+    fi
+    # Fix any misowned subdirectories/files (e.g., from previous runs as root)
+    if [ "$did_full_chown" -eq 0 ] && [ -d "$folder" ]; then
+        echo "Checking for misowned files/directories in $folder"
+        # Stay on the same filesystem to avoid traversing mounted subpaths
+        # (for example read-only bind mounts under /app in dev setups).
+        find "$folder" -xdev -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+            -exec chown "$RUN_UID:$RUN_GID" {} + 2>/dev/null || true
     fi
     test_write $folder || echo "Failed to test write to ${folder}, continuing..."
 }
 
+fix_misowned() {
+    folder=$1
+    mkdir -p $folder
+    echo "Checking for misowned files/directories in $folder"
+    # Stay on the same filesystem to avoid traversing mounted subpaths
+    # (for example read-only bind mounts under /app in dev setups).
+    find "$folder" -xdev \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+        -exec chown "$RUN_UID:$RUN_GID" {} + 2>/dev/null || true
+}
+
 # Ensure proper ownership of application directories
 change_ownership() {
   folder=$1
   mkdir -p $folder
-  echo "Changing ownership of $folder to $USERNAME:$GID"
-  chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
-  chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
+  echo "Changing ownership of $folder to $USERNAME:$RUN_GID"
+  chown -R "${RUN_UID}:${RUN_GID}" "${folder}" || echo "Failed to change ownership for ${folder}, continuing..."
 }
 
-change_ownership /app
-change_ownership /var/log/cwa-book-downloader
-change_ownership /tmp/cwa-book-downloader
+fix_misowned /app
+fix_misowned /var/log/shelfmark
+fix_misowned /tmp/shelfmark
 
-# Test write to all folders
-make_writable /cwa-book-ingest
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
 
-# Set the command to run based on the environment
-is_prod=$(echo "$APP_ENV" | tr '[:upper:]' '[:lower:]')
-if [ "$is_prod" = "prod" ]; then 
-    command="gunicorn -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} app:app"
-else
-    command="python3 app.py"
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
 fi
 
-# IF DEBUG
-if [ "$DEBUG" = "true" ]; then
+# Test write to all folders
+make_writable ${CONFIG_DIR:-/config}
+make_writable ${INGEST_DIR:-/books}
+
+# Fix permissions on directories configured in settings
+echo "Checking for additional configured directories..."
+if [ -f /app/scripts/fix_permissions.py ]; then
+    configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
+    if [ -n "$configured_dirs" ]; then
+        echo "$configured_dirs" | while read -r dir; do
+            if [ -n "$dir" ] && [ -d "$dir" ]; then
+                echo "Checking configured directory: $dir"
+                make_writable "$dir"
+            fi
+        done
+    fi
+fi
+
+# Fallback to root if config dir is still not writable (common on NAS/Unraid after upgrade from v0.4.0)
+CONFIG_PATH=${CONFIG_DIR:-/config}
+set +e
+test_write "$CONFIG_PATH" >/dev/null 2>&1
+config_ok=$?
+set -e
+
+if [ $config_ok -ne 0 ] && [ "$RUN_UID" != "0" ]; then
+    config_owner=$(stat -c '%u' "$CONFIG_PATH" 2>/dev/null || echo "unknown")
+    if [ "$config_owner" = "0" ]; then
+        echo ""
+        echo "========================================================"
+        echo "WARNING: Permission issue detected!"
+        echo ""
+        echo "Config directory is owned by root but PUID=$RUN_UID."
+        echo "This typically happens after upgrading from v0.4.0 where"
+        echo "PUID/PGID settings were not respected."
+        echo ""
+        echo "Falling back to running as root to prevent data loss."
+        echo ""
+        echo "To fix this permanently, run on your HOST machine:"
+        echo "  chown -R $RUN_UID:$RUN_GID /path/to/config"
+        echo ""
+        echo "Then restart the container."
+        echo "========================================================"
+        echo ""
+        RUN_UID=0
+        RUN_GID=0
+        USERNAME=root
+    fi
+fi
+
+# Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
+# upgrades work reliably on customer machines.
+# Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
+gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
+command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
+
+# If DEBUG and not using an external bypass
+if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     set +e
     set -x
     echo "vvvvvvvvvvvv DEBUG MODE vvvvvvvvvvvv"
@@ -166,18 +334,24 @@ if [ "$DEBUG" = "true" ]; then
     echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
 fi
 
-# Hacky way to verify /tmp has at least 1MB of space and is writable/readable
+# Verify /tmp has at least 1MB of space and is writable/readable
 echo "Verifying /tmp has enough space"
-rm -f /tmp/test.cwa-bd
-for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd
-sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))")
-[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1)
-rm /tmp/test.cwa-bd
+rm -f /tmp/test.shelfmark
+if dd if=/dev/zero of=/tmp/test.shelfmark bs=1M count=1 2>/dev/null && \
+   [ "$(wc -c < /tmp/test.shelfmark)" -eq 1048576 ]; then
+    rm -f /tmp/test.shelfmark
+    echo "Success: /tmp is writable and readable"
+else
+    echo "Failure: /tmp is not writable or has insufficient space"
+    exit 1
+fi
 
-echo "Running command: '$command' as '$USERNAME' in '$APP_ENV' mode"
+echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
+# Set umask for file permissions (default: 0022 = files 644, dirs 755)
+UMASK_VALUE=${UMASK:-0022}
+echo "Setting umask to $UMASK_VALUE"
+umask $UMASK_VALUE
 
-exec sudo -E -u "$USERNAME" HOME=/app $command
+stop_file_logging
+exec gosu "$USERNAME" env HOME=/app $command

env.py

@@ -1,55 +0,0 @@
-import os
-from pathlib import Path
-
-def string_to_bool(s: str) -> bool:
-    return s.lower() in ["true", "yes", "1", "y"]
-
-CWA_DB = os.getenv("CWA_DB_PATH")
-CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
-LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
-LOG_DIR = LOG_ROOT / "cwa-book-downloader"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
-INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
-STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
-USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
-MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
-DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
-USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
-HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
-HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
-AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
-_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
-_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
-_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
-_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
-_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
-FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
-FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
-PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
-
-# If debug is true, we want to log everything
-if DEBUG:
-    LOG_LEVEL = "DEBUG"
-else:
-    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
-MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
-MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "5"))
-DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip()
-USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
-BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
-APP_ENV = os.getenv("APP_ENV", "prod").lower()
-# Logging settings
-LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
-
-USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
-# If using Tor, we don't need to set custom DNS, use DOH, or proxy
-if USING_TOR:
-    _CUSTOM_DNS = ""
-    USE_DOH = False
-    HTTP_PROXY = ""
-    HTTPS_PROXY = ""
-    

genDebug.sh

@@ -2,8 +2,8 @@
 
 # Set up log paths
 LOG_ROOT=${LOG_ROOT:-"/var/log"}
-LOG_DIR="$LOG_ROOT/cwa-book-downloader"
-OUTPUT_FILE_NAME="cwa-book-downloader-debug_BUILD-${BUILD_VERSION:-local}_$(date +%Y%m%d-%H%M%S)"
+LOG_DIR="$LOG_ROOT/shelfmark"
+OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
 OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
 
 # Create LOG_DIR if it doesn't exist
@@ -18,17 +18,17 @@ echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add disk usage
 echo "=== Disk Usage ===" >> "$LOG_DIR/system_info.txt"
-df -h >> "$LOG_DIR/system_info.txt"
+df -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add memory info
 echo "=== Memory Info ===" >> "$LOG_DIR/system_info.txt"
-free -h >> "$LOG_DIR/system_info.txt"
+free -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add running processes
 echo "=== Running Processes ===" >> "$LOG_DIR/system_info.txt"
-ps aux >> "$LOG_DIR/system_info.txt"
+ps aux >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"
 
 # Add network information using basic commands
@@ -37,17 +37,17 @@ echo "=== Network Information ===" > "$LOG_DIR/network_info.txt"
 # Try to get basic connectivity information
 echo "=== Basic Connectivity ===" >> "$LOG_DIR/network_info.txt"
 echo "Hostname resolution:" >> "$LOG_DIR/network_info.txt"
-cat /etc/hosts 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
+cat /etc/hosts >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 echo "DNS configuration:" >> "$LOG_DIR/network_info.txt"
-cat /etc/resolv.conf 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
+cat /etc/resolv.conf >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try to get interface information from /proc
 echo "=== Network Interfaces (/proc) ===" >> "$LOG_DIR/network_info.txt"
 if [ -f "/proc/net/dev" ]; then
-  cat /proc/net/dev >> "$LOG_DIR/network_info.txt"
+  cat /proc/net/dev >> "$LOG_DIR/network_info.txt" 2>&1
 else
   echo "Not available: /proc/net/dev not found" >> "$LOG_DIR/network_info.txt"
 fi
@@ -55,9 +55,9 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try connectivity tests
 echo "=== Internet Connectivity ===" >> "$LOG_DIR/network_info.txt"
-ping -c 3 1.1.1.1 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
+ping -c 3 1.1.1.1 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
-ping -c 3 one.one.one.one 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
+ping -c 3 one.one.one.one >> "$LOG_DIR/network_info.txt" 2>&1 || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test IPv6 connectivity
@@ -77,7 +77,7 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Try IPv6 connectivity test using Cloudflare's IPv6 DNS
 echo "Testing IPv6 connectivity to Cloudflare DNS:" >> "$LOG_DIR/network_info.txt"
-ping6 -c 3 2606:4700:4700::1111 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
+ping6 -c 3 2606:4700:4700::1111 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
 
 # Test SSL connectivity
@@ -92,24 +92,36 @@ echo "" >> "$LOG_DIR/network_info.txt"
 
 # Add installed packages
 echo "=== Installed Python Packages ===" > "$LOG_DIR/packages.txt"
-pip list 2>/dev/null >> "$LOG_DIR/packages.txt" || echo "pip not found" >> "$LOG_DIR/packages.txt"
+pip list >> "$LOG_DIR/packages.txt" 2>&1 || echo "pip not found" >> "$LOG_DIR/packages.txt"
 echo "" >> "$LOG_DIR/packages.txt"
 
 # Check Permissions
 echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
-ls -all /app >> "$LOG_DIR/permissions.txt"
+ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt"
+echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
+ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 
+# Check Iptables (NAT)
+echo "=== IPtables NAT Rules ===" > "$LOG_DIR/iptables_nat.txt"
+iptables -t nat -L -v -n >> "$LOG_DIR/iptables_nat.txt" 2>&1
+
+# Check DNS Resolution details
+echo "=== DNS Resolution Test ===" > "$LOG_DIR/dns_test.txt"
+echo "Resolving google.com:" >> "$LOG_DIR/dns_test.txt"
+nslookup google.com >> "$LOG_DIR/dns_test.txt" 2>&1
+echo "" >> "$LOG_DIR/dns_test.txt"
+echo "Resolving check.torproject.org:" >> "$LOG_DIR/dns_test.txt"
+nslookup check.torproject.org >> "$LOG_DIR/dns_test.txt" 2>&1
+
 
 # Check if running in Docker
 echo "=== Container Info ===" > "$LOG_DIR/container_info.txt"
@@ -122,19 +134,58 @@ else
 fi
 
 # Add environment variables (redacting sensitive info)
-env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt"
+env | grep -v -E "(AA_DONATOR_KEY|HARDCOVER_API_KEY|_KEY=|_SECRET=|_PASSWORD=|_TOKEN=)" | sort > "$LOG_DIR/environment.txt"
 
-echo "--- HTTPBin ---" > $LOG_DIR/network_info.txt
-pyrequests https://httpbin.org/get >> $LOG_DIR/network_info.txt
-ehco ""
+# Add configuration files (redacting sensitive values)
+CONFIG_DIR=${CONFIG_DIR:-"/config"}
+if [ -d "$CONFIG_DIR" ]; then
+  mkdir -p "$LOG_DIR/config"
+
+  # Copy and redact main settings file
+  if [ -f "$CONFIG_DIR/settings.json" ]; then
+    # Redact sensitive fields (API keys, passwords, tokens)
+    sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+      "$CONFIG_DIR/settings.json" > "$LOG_DIR/config/settings.json" 2>/dev/null
+  fi
+
+  # Copy and redact plugin config files
+  if [ -d "$CONFIG_DIR/plugins" ]; then
+    mkdir -p "$LOG_DIR/config/plugins"
+    for config_file in "$CONFIG_DIR/plugins"/*.json; do
+      if [ -f "$config_file" ]; then
+        filename=$(basename "$config_file")
+        sed -E 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
+          "$config_file" > "$LOG_DIR/config/plugins/$filename" 2>/dev/null
+      fi
+    done
+  fi
+
+  echo "Configuration files copied (sensitive values redacted)" >> "$LOG_DIR/container_info.txt"
+else
+  echo "Config directory not found at $CONFIG_DIR" >> "$LOG_DIR/container_info.txt"
+fi
+
+echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
+curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
-pyrequests https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt
-ehco ""
+curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
-pyrequests https://ipinfo.io >> $LOG_DIR/network_info.txt
-ehco ""
+curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt 2>&1
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
-pyrequests https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt
+curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt 2>&1
+
+# Copy Tor logs if they exist
+if [ -f "/var/log/tor/notices.log" ]; then
+  cp "/var/log/tor/notices.log" "$LOG_DIR/tor_notices.log"
+fi
+
+# Copy Supervisor logs if they exist
+if [ -d "/var/log/supervisor" ]; then
+  cp -rf "/var/log/supervisor/" "$LOG_DIR/supervisor/"
+fi
 
 # Create the zip file directly from LOG_DIR
 ln -s "$LOG_DIR" /tmp/$OUTPUT_FILE_NAME

models.py

@@ -1,348 +0,0 @@
-"""Data structures and models used across the application."""
-
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Tuple
-from enum import Enum
-from datetime import datetime, timedelta
-from threading import Lock, Event
-from pathlib import Path
-import queue
-import time
-from env import INGEST_DIR, STATUS_TIMEOUT
-
-class QueueStatus(str, Enum):
-    """Enum for possible book queue statuses."""
-    QUEUED = "queued"
-    DOWNLOADING = "downloading"
-    AVAILABLE = "available"
-    ERROR = "error"
-    DONE = "done"
-    CANCELLED = "cancelled"
-
-@dataclass
-class QueueItem:
-    """Queue item with priority and metadata."""
-    book_id: str
-    priority: int
-    added_time: float
-    
-    def __lt__(self, other):
-        """Compare items for priority queue (lower priority number = higher precedence)."""
-        if self.priority != other.priority:
-            return self.priority < other.priority
-        return self.added_time < other.added_time
-
-@dataclass
-class BookInfo:
-    """Data class representing book information."""
-    id: str
-    title: str
-    preview: Optional[str] = None
-    author: Optional[str] = None
-    publisher: Optional[str] = None
-    year: Optional[str] = None
-    language: Optional[str] = None
-    format: Optional[str] = None
-    size: Optional[str] = None
-    info: Optional[Dict[str, List[str]]] = None
-    download_urls: List[str] = field(default_factory=list)
-    download_path: Optional[str] = None
-    priority: int = 0
-    progress: Optional[float] = None
-
-class BookQueue:
-    """Thread-safe book queue manager with priority support and cancellation."""
-    def __init__(self) -> None:
-        self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
-        self._lock = Lock()
-        self._status: dict[str, QueueStatus] = {}
-        self._book_data: dict[str, BookInfo] = {}
-        self._status_timestamps: dict[str, datetime] = {}  # Track when each status was last updated
-        self._status_timeout = timedelta(seconds=STATUS_TIMEOUT)  # 1 hour timeout
-        self._cancel_flags: dict[str, Event] = {}  # Cancellation flags for active downloads
-        self._active_downloads: dict[str, bool] = {}  # Track currently downloading books
-    
-    def add(self, book_id: str, book_data: BookInfo, priority: int = 0) -> None:
-        """Add a book to the queue with specified priority.
-        
-        Args:
-            book_id: Unique identifier for the book
-            book_data: Book information
-            priority: Priority level (lower number = higher priority)
-        """
-        with self._lock:
-            # Don't add if already exists and not in error/done state
-            if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                return
-                
-            book_data.priority = priority
-            queue_item = QueueItem(book_id, priority, time.time())
-            self._queue.put(queue_item)
-            self._book_data[book_id] = book_data
-            self._update_status(book_id, QueueStatus.QUEUED)
-    
-    def get_next(self) -> Optional[Tuple[str, Event]]:
-        """Get next book ID from queue with cancellation flag.
-        
-        Returns:
-            Tuple of (book_id, cancel_flag) or None if queue is empty
-        """
-        try:
-            queue_item = self._queue.get_nowait()
-            book_id = queue_item.book_id
-            
-            with self._lock:
-                # Check if book was cancelled while in queue
-                if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED:
-                    return self.get_next()  # Recursively get next non-cancelled item
-                
-                # Create cancellation flag for this download
-                cancel_flag = Event()
-                self._cancel_flags[book_id] = cancel_flag
-                self._active_downloads[book_id] = True
-                
-            return book_id, cancel_flag
-        except queue.Empty:
-            return None
-            
-    def _update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Internal method to update status and timestamp."""
-        self._status[book_id] = status
-        self._status_timestamps[book_id] = datetime.now()
-            
-    def update_status(self, book_id: str, status: QueueStatus) -> None:
-        """Update status of a book in the queue."""
-        with self._lock:
-            self._update_status(book_id, status)
-            
-            # Clean up active download tracking when finished
-            if status in [QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
-                self._active_downloads.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-    
-    def update_download_path(self, book_id: str, download_path: str) -> None:
-        """Update the download path of a book in the queue."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].download_path = download_path
-                
-    def update_progress(self, book_id: str, progress: float) -> None:
-        """Update download progress for a book."""
-        with self._lock:
-            if book_id in self._book_data:
-                self._book_data[book_id].progress = progress
-            
-    def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]:
-        """Get current queue status."""
-        self.refresh()
-        with self._lock:
-            result: Dict[QueueStatus, Dict[str, BookInfo]] = {status: {} for status in QueueStatus}
-            for book_id, status in self._status.items():
-                if book_id in self._book_data:
-                    result[status][book_id] = self._book_data[book_id]
-            return result
-            
-    def get_queue_order(self) -> List[Dict[str, any]]:
-        """Get current queue order for display."""
-        with self._lock:
-            queue_items = []
-            
-            # Get items from priority queue without removing them
-            temp_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    temp_items.append(item)
-                    if item.book_id in self._book_data:
-                        book_info = self._book_data[item.book_id]
-                        queue_items.append({
-                            'id': item.book_id,
-                            'title': book_info.title,
-                            'author': book_info.author,
-                            'priority': item.priority,
-                            'added_time': item.added_time,
-                            'status': self._status.get(item.book_id, QueueStatus.QUEUED)
-                        })
-                except queue.Empty:
-                    break
-            
-            # Put items back in queue
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
-            
-    def cancel_download(self, book_id: str) -> bool:
-        """Cancel a download and mark it as cancelled.
-        
-        Args:
-            book_id: Book identifier to cancel
-            
-        Returns:
-            bool: True if cancellation was successful
-        """
-        with self._lock:
-            current_status = self._status.get(book_id)
-            
-            if current_status == QueueStatus.DOWNLOADING:
-                # Signal active download to stop
-                if book_id in self._cancel_flags:
-                    self._cancel_flags[book_id].set()
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            elif current_status == QueueStatus.QUEUED:
-                # Remove from queue and mark as cancelled
-                self._update_status(book_id, QueueStatus.CANCELLED)
-                return True
-            
-            return False
-            
-    def set_priority(self, book_id: str, new_priority: int) -> bool:
-        """Change the priority of a queued book.
-        
-        Args:
-            book_id: Book identifier
-            new_priority: New priority level (lower = higher priority)
-            
-        Returns:
-            bool: True if priority was successfully changed
-        """
-        with self._lock:
-            if book_id not in self._status or self._status[book_id] != QueueStatus.QUEUED:
-                return False
-                
-            # Remove book from queue and re-add with new priority
-            temp_items = []
-            found = False
-            
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    if item.book_id == book_id:
-                        # Create new item with updated priority
-                        new_item = QueueItem(book_id, new_priority, item.added_time)
-                        temp_items.append(new_item)
-                        found = True
-                        # Update book data priority
-                        if book_id in self._book_data:
-                            self._book_data[book_id].priority = new_priority
-                    else:
-                        temp_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back
-            for item in temp_items:
-                self._queue.put(item)
-                
-            return found
-            
-    def reorder_queue(self, book_priorities: Dict[str, int]) -> bool:
-        """Bulk reorder queue by setting new priorities.
-        
-        Args:
-            book_priorities: Dict mapping book_id to new priority
-            
-        Returns:
-            bool: True if reordering was successful
-        """
-        with self._lock:
-            # Extract all items from queue
-            all_items = []
-            while not self._queue.empty():
-                try:
-                    item = self._queue.get_nowait()
-                    # Update priority if specified
-                    if item.book_id in book_priorities:
-                        new_priority = book_priorities[item.book_id]
-                        item = QueueItem(item.book_id, new_priority, item.added_time)
-                        # Update book data priority
-                        if item.book_id in self._book_data:
-                            self._book_data[item.book_id].priority = new_priority
-                    all_items.append(item)
-                except queue.Empty:
-                    break
-            
-            # Put all items back with updated priorities
-            for item in all_items:
-                self._queue.put(item)
-                
-            return True
-            
-    def get_active_downloads(self) -> List[str]:
-        """Get list of currently active download book IDs."""
-        with self._lock:
-            return list(self._active_downloads.keys())
-            
-    def clear_completed(self) -> int:
-        """Remove all completed, errored, or cancelled books from tracking.
-        
-        Returns:
-            int: Number of books removed
-        """
-        with self._lock:
-            to_remove = []
-            for book_id, status in self._status.items():
-                if status in [QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
-                    to_remove.append(book_id)
-            
-            removed_count = len(to_remove)
-            for book_id in to_remove:
-                self._status.pop(book_id, None)
-                self._status_timestamps.pop(book_id, None)
-                self._book_data.pop(book_id, None)
-                self._cancel_flags.pop(book_id, None)
-                self._active_downloads.pop(book_id, None)
-                
-            return removed_count
-        
-    def refresh(self) -> None:
-        """Remove any books that are done downloading or have stale status."""
-        with self._lock:
-            current_time = datetime.now()
-            
-            # Create a list of items to remove to avoid modifying dict during iteration
-            to_remove = []
-            
-            for book_id, status in self._status.items():
-                path = self._book_data[book_id].download_path
-                if path and not Path(path).exists():
-                    self._book_data[book_id].download_path = None
-                    path = None
-                
-                # Check for completed downloads
-                if status == QueueStatus.AVAILABLE:
-                    if not path:
-                        self._update_status(book_id, QueueStatus.DONE)
-                
-                # Check for stale status entries
-                last_update = self._status_timestamps.get(book_id)
-                if last_update and (current_time - last_update) > self._status_timeout:
-                    if status in [QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]:
-                        to_remove.append(book_id)
-            
-            # Remove stale entries
-            for book_id in to_remove:
-                del self._status[book_id]
-                del self._status_timestamps[book_id]
-                if book_id in self._book_data:
-                    del self._book_data[book_id]
-
-    def set_status_timeout(self, hours: int) -> None:
-        """Set the status timeout duration in hours."""
-        with self._lock:
-            self._status_timeout = timedelta(hours=hours)
-
-
-# Global instance of BookQueue
-book_queue = BookQueue()
-
-@dataclass
-class SearchFilters:
-    isbn: Optional[List[str]] = None
-    author: Optional[List[str]] = None
-    title: Optional[List[str]] = None
-    lang: Optional[List[str]] = None
-    sort: Optional[str] = None
-    content: Optional[List[str]] = None
-    format: Optional[List[str]] = None

network.py

@@ -1,350 +0,0 @@
-"""Network operations manager for the book downloader application."""
-
-import requests
-import urllib.request
-from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable
-import socket
-import dns.resolver
-from socket import AddressFamily, SocketKind
-import urllib.parse
-import ssl
-import ipaddress
-
-from logger import setup_logger
-from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER
-import config
-
-logger = setup_logger(__name__)
-
-# Common helper functions for DNS resolution
-def _decode_host(host: Union[str, bytes, None]) -> str:
-    """Convert host to string, handling bytes and None cases."""
-    if host is None:
-        return ""
-    if isinstance(host, bytes):
-        return host.decode('utf-8')
-    return str(host)
-
-def _decode_port(port: Union[str, bytes, int, None]) -> int:
-    """Convert port to integer, handling various input types."""
-    if port is None:
-        return 0
-    if isinstance(port, (str, bytes)):
-        return int(port)
-    return int(port)
-
-def _is_local_address(host_str: str) -> bool:
-    """Check if an address is local and should bypass custom DNS."""
-    """Check if an address is local or private and should bypass custom DNS."""
-    # Localhost checks
-    if (host_str == 'localhost' or 
-        host_str.startswith('127.') or 
-        host_str == '::1' or 
-        host_str == '0.0.0.0'):
-        return True
-        
-    # IPv4 private ranges (RFC 1918)
-    if (host_str.startswith('10.') or 
-        (host_str.startswith('172.') and 
-         len(host_str.split('.')) > 1 and 
-         16 <= int(host_str.split('.')[1]) <= 31) or
-        host_str.startswith('192.168.')):
-        return True
-        
-    # IPv6 private ranges
-    if (host_str.startswith('fc') or 
-        host_str.startswith('fd') or  # Unique local addresses (fc00::/7)
-        host_str.startswith('fe80:')):  # Link-local addresses (fe80::/10)
-        return True
-    
-    return False
-
-def _is_ip_address(host_str: str) -> bool:
-    """Check if a string is a valid IP address (IPv4 or IPv6)."""
-    try:
-        ipaddress.ip_address(host_str)
-        return True
-    except ValueError:
-        return False
-
-# Store the original getaddrinfo function
-original_getaddrinfo = socket.getaddrinfo
-
-class DoHResolver:
-    """DNS over HTTPS resolver implementation."""
-    def __init__(self, provider_url: str, hostname: str, ip: str):
-        """Initialize DoH resolver with specified provider."""
-        self.base_url = provider_url.lower().strip()
-        self.hostname = hostname  # Store the hostname for hostname-based skipping
-        self.ip = ip              # Store IP for direct connections
-        self.session = requests.Session()
-        
-        # Different headers based on provider
-        if 'google' in self.base_url:
-            self.session.headers.update({
-                'Accept': 'application/json',
-            })
-        else:
-            self.session.headers.update({
-                'Accept': 'application/dns-json',
-            })
-    
-    def resolve(self, hostname: str, record_type: str) -> List[str]:
-        """Resolve a hostname using DoH.
-        
-        Args:
-            hostname: The hostname to resolve
-            record_type: The DNS record type (A or AAAA)
-            
-        Returns:
-            List of resolved IP addresses
-        """
-        # Check if hostname is already an IP address, no need to resolve
-        if _is_ip_address(hostname):
-            logger.debug(f"Skipping DoH resolution for IP address: {hostname}")
-            return [hostname]
-            
-        # Check if hostname is a private IP address, and skip DoH if it is
-        if _is_local_address(hostname):
-            logger.debug(f"Skipping DoH resolution for private IP: {hostname}")
-            return [hostname]
-            
-        # Skip resolution for the DoH server itself to prevent recursion
-        if hostname == self.hostname:
-            logger.debug(f"Skipping DoH resolution for DoH server itself: {hostname}")
-            return [self.ip]
-            
-        try:
-            params = {
-                'name': hostname,
-                'type': 'AAAA' if record_type == 'AAAA' else 'A'
-            }
-            
-            response = self.session.get(
-                self.base_url,
-                params=params,
-                proxies=PROXIES,
-                timeout=5
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            if 'Answer' not in data:
-                logger.warning(f"DoH resolution failed for {hostname}: {data}")
-                return []
-            
-            # Extract IP addresses from the response    
-            answers = [answer['data'] for answer in data['Answer'] 
-                    if answer.get('type') == (28 if record_type == 'AAAA' else 1)]
-            logger.debug(f"Resolved {hostname} to {len(answers)} addresses using DoH: {answers}")
-            return answers
-            
-        except Exception as e:
-            logger.warning(f"DoH resolution failed for {hostname}: {e}")
-            return []
-
-def create_custom_resolver():
-    """Create a custom DNS resolver using the configured DNS servers."""
-    custom_resolver = dns.resolver.Resolver()
-    custom_resolver.nameservers = CUSTOM_DNS
-    return custom_resolver
-
-def resolve_with_custom_dns(resolver, hostname: str, record_type: str) -> List[str]:
-    """Resolve hostname using custom DNS resolver.
-    
-    Args:
-        resolver: The DNS resolver to use
-        hostname: The hostname to resolve
-        record_type: The DNS record type (A or AAAA)
-        
-    Returns:
-        List of resolved IP addresses
-    """
-    try:
-        answers = resolver.resolve(hostname, record_type)
-        return [str(answer) for answer in answers]
-    except Exception as e:
-        logger.debug(f"{record_type} resolution failed for {hostname}: {e}")
-        return []
-
-def create_custom_getaddrinfo(
-    resolve_ipv4: Callable[[str], List[str]],
-    resolve_ipv6: Callable[[str], List[str]],
-    skip_check: Optional[Callable[[str], bool]] = None
-):
-    """Create a custom getaddrinfo function that uses the provided resolvers.
-    
-    Args:
-        resolve_ipv4: Function to resolve IPv4 addresses
-        resolve_ipv6: Function to resolve IPv6 addresses
-        skip_check: Optional function to check if custom resolution should be skipped
-        
-    Returns:
-        A custom getaddrinfo function
-    """
-    def custom_getaddrinfo(
-        host: Union[str, bytes, None],
-        port: Union[str, bytes, int, None],
-        family: int = 0,
-        type: int = 0,
-        proto: int = 0,
-        flags: int = 0
-    ) -> Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]]:
-        host_str = _decode_host(host)
-        port_int = _decode_port(port)
-        
-        # Skip custom resolution for IP addresses, local addresses, or if skip check passes
-        if _is_ip_address(host_str) or _is_local_address(host_str) or (skip_check and skip_check(host_str)):
-            logger.debug(f"Using system DNS for IP address or local/private address: {host_str}")
-            return original_getaddrinfo(host, port, family, type, proto, flags)
-        
-        results: list[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]] = []
-        
-        try:
-            # Try IPv6 first if family allows it
-            if family == 0 or family == socket.AF_INET6:
-                logger.debug(f"Resolving IPv6 address for {host_str}")
-                ipv6_answers = resolve_ipv6(host_str)
-                for answer in ipv6_answers:
-                    results.append((socket.AF_INET6, cast(SocketKind, type), proto, '', (answer, port_int, 0, 0)))
-                if ipv6_answers:
-                    logger.debug(f"Found {len(ipv6_answers)} IPv6 addresses for {host_str}")
-            
-            # Then try IPv4
-            if family == 0 or family == socket.AF_INET:
-                logger.debug(f"Resolving IPv4 address for {host_str}")
-                ipv4_answers = resolve_ipv4(host_str)
-                for answer in ipv4_answers:
-                    results.append((socket.AF_INET, cast(SocketKind, type), proto, '', (answer, port_int)))
-                if ipv4_answers:
-                    logger.debug(f"Found {len(ipv4_answers)} IPv4 addresses for {host_str}")
-            
-            if results:
-                logger.debug(f"Resolved {host_str} to {len(results)} addresses")
-                return results
-                
-        except Exception as e:
-            logger.warning(f"Custom DNS resolution failed for {host_str}: {e}, falling back to system DNS")
-        
-        # Fall back to system DNS if custom resolution fails
-        try:
-            return original_getaddrinfo(host, port, family, type, proto, flags)
-        except Exception as e:
-            logger.error(f"System DNS resolution also failed for {host_str}: {e}")
-            # Last resort: Try to connect to the hostname directly
-            if family == 0 or family == socket.AF_INET:
-                logger.warning(f"Using direct hostname as last resort for {host_str}")
-                return [(socket.AF_INET, cast(SocketKind, type), proto, '', (host_str, port_int))]
-            else:
-                raise  # Re-raise the exception if we can't provide a last resort
-    
-    return custom_getaddrinfo
-
-def init_doh_resolver(doh_server: str = DOH_SERVER):
-    """Initialize DNS over HTTPS resolver.
-    
-    Args:
-        doh_server: The DoH server URL
-    """
-    # Pre-resolve the DoH server hostname to prevent recursion
-    url = urllib.parse.urlparse(doh_server)
-    server_hostname = url.hostname if url.hostname else ''
-    
-    # Use system DNS for DoH server to prevent circular dependencies
-    try:
-        # Temporarily restore original getaddrinfo to resolve DoH server
-        temp_getaddrinfo = socket.getaddrinfo
-        socket.getaddrinfo = original_getaddrinfo
-        
-        server_ip = socket.gethostbyname(server_hostname)
-        logger.info(f"DoH server {server_hostname} resolved to IP: {server_ip}")
-        
-        # Restore custom getaddrinfo if it was previously set
-        socket.getaddrinfo = temp_getaddrinfo
-    except Exception as e:
-        logger.error(f"Failed to resolve DoH server {server_hostname}: {e}")
-        # Fall back to a known public DNS if resolution fails
-        server_ip = "1.1.1.1"
-        logger.info(f"Using fallback IP for DoH server: {server_ip}")
-    
-    # Create DoH resolver
-    doh_resolver = DoHResolver(doh_server, server_hostname, server_ip)
-    
-    # Create resolver functions
-    def resolve_ipv4(hostname: str) -> List[str]:
-        return doh_resolver.resolve(hostname, 'A')
-    
-    def resolve_ipv6(hostname: str) -> List[str]:
-        return doh_resolver.resolve(hostname, 'AAAA')
-    
-    # Skip DoH resolution for the DoH server itself, IP addresses, and private addresses
-    def skip_doh(hostname: str) -> bool:
-        return (hostname == server_hostname or 
-                hostname == server_ip or 
-                _is_ip_address(hostname) or 
-                _is_local_address(hostname))
-    
-    # Replace socket.getaddrinfo with our DoH-enabled version
-    socket.getaddrinfo = cast(Any, create_custom_getaddrinfo(
-        resolve_ipv4, resolve_ipv6, skip_doh
-    ))
-    
-    logger.info("DoH resolver successfully configured and activated")
-    return doh_resolver
-
-def init_custom_resolver():
-    """Initialize custom DNS resolver using configured DNS servers."""
-    custom_resolver = create_custom_resolver()
-    
-    # Create resolver functions
-    def resolve_ipv4(hostname: str) -> List[str]:
-        return resolve_with_custom_dns(custom_resolver, hostname, 'A')
-    
-    def resolve_ipv6(hostname: str) -> List[str]:
-        return resolve_with_custom_dns(custom_resolver, hostname, 'AAAA')
-    
-    # Replace socket.getaddrinfo with our custom resolver
-    socket.getaddrinfo = cast(Any, create_custom_getaddrinfo(resolve_ipv4, resolve_ipv6))
-    
-    logger.info("Custom DNS resolver successfully configured and activated")
-    return custom_resolver
-
-# Initialize DNS resolvers based on configuration
-def init_dns_resolvers():
-    """Initialize DNS resolvers based on configuration."""
-    if len(CUSTOM_DNS) > 0:
-        init_custom_resolver()
-        if DOH_SERVER:
-            init_doh_resolver()
-
-# Initialize DNS resolvers
-init_dns_resolvers()
-
-# Check available AA_BASE_URLs if set to auto
-if AA_BASE_URL == "auto":
-    logger.info(f"AA_BASE_URL: auto, checking available urls {AA_AVAILABLE_URLS}")
-    for url in AA_AVAILABLE_URLS:
-        try:
-            response = requests.get(url, proxies=PROXIES)
-            if response.status_code == 200:
-                AA_BASE_URL = url
-                break
-        except Exception as e:
-            logger.error_trace(f"Error checking {url}: {e}")
-    if AA_BASE_URL == "auto":
-        AA_BASE_URL = AA_AVAILABLE_URLS[0]
-config.AA_BASE_URL = AA_BASE_URL
-logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
-
-# Configure urllib opener with appropriate headers
-opener = urllib.request.build_opener()
-opener.addheaders = [
-    ('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        'AppleWebKit/537.36 (KHTML, like Gecko) '
-        'Chrome/129.0.0.0 Safari/537.3')
-]
-urllib.request.install_opener(opener)
-
-# Need an empty function to be called by downloader.py
-def init():
-    pass

package-lock.json

@@ -0,0 +1,22 @@
+{
+  "name": "shelfmark",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "devDependencies": {
+        "baseline-browser-mapping": "^2.9.19"
+      }
+    },
+    "node_modules/baseline-browser-mapping": {
+      "version": "2.9.19",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz",
+      "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "baseline-browser-mapping": "dist/cli.js"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "baseline-browser-mapping": "^2.9.19"
+  }
+}

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "shelfmark"
+version = "0.1.0"
+description = "Shelfmark - Book Downloader"
+requires-python = ">=3.10"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--tb=short",
+]
+markers = [
+    "integration: marks tests that require running services (deselect with '-m \"not integration\"')",
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks end-to-end tests that require the full application stack",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+ignore_missing_imports = true

readme.md

@@ -1,256 +1,256 @@
-# 📚 Calibre-Web-Automated-Book-Downloader
+# 📚 Shelfmark: Book Downloader
 
-![Calibre-Web Automated Book Downloader](static/media/logo.png 'Calibre-Web Automated Book Downloader')
+Formerly *Calibre Web Automated Book Downloader (CWABD)*
 
-An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
+<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
+
+Shelfmark is a self-hosted web interface for searching and downloading books and audiobooks from multiple sources. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
+
+**Fully standalone** - no external dependencies required. Works great alongside the following library tools, with support for automatic imports:
+- [Calibre](https://calibre-ebook.com/)
+- [Calibre-Web](https://github.com/janeczku/calibre-web)
+- [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated)
+- [Booklore](https://github.com/booklore-app/booklore)
+- [Audiobookshelf](https://github.com/advplyr/audiobookshelf)
 
 ## ✨ Features
 
-- 🌐 User-friendly web interface for book search and download
-- 🔄 Automated download to your specified ingest folder
-- 🔌 Seamless integration with Calibre-Web-Automated
-- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr)
-- 🛡️ Cloudflare bypass capability for reliable downloads
-- 🐳 Docker-based deployment for quick setup
+- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
+- **Multiple Sources** - Popular archive websites, Torrent, Usenet, and IRC download support
+- **Audiobook Support** - Full audiobook search and download with dedicated processing
+- **Two Search Modes**:
+  - **Direct** - Search popular web sources
+  - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
+- **Multi-User & Requests** - Share your instance with others, let users browse and request books, and manage approvals with configurable notifications
+- **Authentication** - Built-in login, OIDC single sign-on, proxy auth, and Calibre-Web database support
+- **Real-Time Progress** - Unified download queue with live status updates across all sources
+- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
 
 ## 🖼️ Screenshots
 
-![Main search interface Screenshot](README_images/search.png 'Main search interface')
+**Home screen**
+![Home screen](README_images/homescreen.png 'Home screen')
 
-![Details modal Screenshot placeholder](README_images/details.png 'Details modal')
+**Search results**
+![Search results](README_images/search-results.png 'Search results')
 
-![Download queue Screenshot placeholder](README_images/downloading.png 'Download queue')
+**Multi-source downloads**
+![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
+
+**Download queue**
+![Download queue](README_images/downloads.png 'Download queue')
 
 ## 🚀 Quick Start
 
 ### Prerequisites
 
-- Docker
-- Docker Compose
-- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended)
+- Docker & Docker Compose
 
-### Installation Steps
-
-1. Get the docker-compose.yml:
+### Installation
 
+1. Download the [docker-compose file](compose/docker-compose.yml):
    ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml
+   curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
    ```
 
 2. Start the service:
-
    ```bash
    docker compose up -d
    ```
 
-3. Access the web interface at `http://localhost:8084`
+3. Open `http://localhost:8084`
 
-## ⚙️ Configuration
+That's it! Configure settings through the web interface as needed.
 
-### Environment Variables
-
-#### Application Settings
-
-| Variable          | Description             | Default Value      |
-| ----------------- | ----------------------- | ------------------ |
-| `FLASK_PORT`      | Web interface port      | `8084`             |
-| `FLASK_HOST`      | Web interface binding   | `0.0.0.0`          |
-| `DEBUG`           | Debug mode toggle       | `false`            |
-| `INGEST_DIR`      | Book download directory | `/cwa-book-ingest` |
-| `TZ`              | Container timezone      | `UTC`              |
-| `UID`             | Runtime user ID         | `1000`             |
-| `GID`             | Runtime group ID        | `100`              |
-| `CWA_DB_PATH`     | Calibre-Web's database  | None               |
-| `ENABLE_LOGGING`  | Enable log file         | `true`             |
-| `LOG_LEVEL`       | Log level to use        | `info`             |
-
-If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
-
-If logging is enabld, log folder default location is `/var/log/cwa-book-downloader`
-Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages.
-
-Note that if using TOR, the TZ will be calculated automatically based on IP.
-
-#### Download Settings
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `MAX_RETRY`            | Maximum retry attempts                                    | `3`                               |
-| `DEFAULT_SLEEP`        | Retry delay (seconds)                                     | `5`                               |
-| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds)                           | `5`                               |
-| `SUPPORTED_FORMATS`    | Supported book formats                                    | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
-| `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
-| `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
-| `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
-| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
-
-If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
-
-#### AA 
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
-
-If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
-If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.
-
-#### Network Settings
-
-| Variable               | Description                     | Default Value           |
-| ---------------------- | ------------------------------- | ----------------------- |
-| `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
-| `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
-| `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | Custom DNS IP                   | ``                      |
-| `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
-
-For proxy configuration, you can specify URLs in the following format:
-```bash
-# Basic proxy
-HTTP_PROXY=http://proxy.example.com:8080
-HTTPS_PROXY=http://proxy.example.com:8080
-
-# Proxy with authentication
-HTTP_PROXY=http://username:password@proxy.example.com:8080
-HTTPS_PROXY=http://username:password@proxy.example.com:8080
-```
-
-
-The `CUSTOM_DNS` setting supports two formats:
-
-1. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
-   - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses in the same string
-
-2. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS
-   - `quad9` - Quad9 DNS
-   - `cloudflare` - Cloudflare DNS
-   - `opendns` - OpenDNS
-
-For users experiencing ISP-level website blocks (such as Virgin Media in the UK), using alternative DNS providers like Cloudflare may help bypass these restrictions
-
-If a `CUSTOM_DNS` is specified from the preset providers, you can also set a `USE_DOH=true` to force using DNS over HTTPS,
-which might also help in certain network situations. Note that only `google`, `quad9`, `cloudflare` and `opendns` are 
-supported for now, and any other value in `CUSTOM_DNS` will make the `USE_DOH` flag ignored.
-
-Try something like this :
-```bash
-CUSTOM_DNS=cloudflare
-USE_DOH=true
-```
-
-#### Custom configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `CUSTOM_SCRIPT`        | Path to an executable script that tuns after each download  | ``                      |
-
-If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation.
-
-The script is called with the full path of the downloaded file as its argument. Important notes:
-- The script must preserve the original filename for proper processing
-- The file can be modified or even deleted if needed
-- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted)
-
-You can specify these configuration in this format :
-```
-environment:
-  - CUSTOM_SCRIPT=/scripts/process-book.sh
-
-volumes:
-  - local/scripts/custom_script.sh:/scripts/process-book.sh
-```
-
-### Volume Configuration
+### Volume Setup
 
 ```yaml
 volumes:
-  - /your/local/path:/cwa-book-ingest
-  - /cwa/config/path/app.db:/auth/app.db:ro
-```
-**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777
-
-Mount should align with your Calibre-Web-Automated ingest folder.
-
-## 🧅 Tor Variant
-
-This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions.
-
-To use the Tor variant:
-
-1.  Get the Tor-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.tor.yml up -d
-    ```
-
-**Important Considerations for Tor:**
-
-*   **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying.
-*   **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set.
-*   **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor.
-
-## 🏗️ Architecture
-
-The application consists of a single service:
-
-1. **calibre-web-automated-bookdownloader**: Main application providing web interface and download functionality
-
-## 🏥 Health Monitoring
-
-Built-in health checks monitor:
-
-- Web interface availability
-- Download service status
-- Cloudflare bypass service connection
-
-Checks run every 30 seconds with a 30-second timeout and 3 retries.
-You can enable by adding this to your compose :
-```
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD pyrequests http://localhost:8084/request/api/status || exit 1
+  - /your/config/path:/config # Config, database, and artwork cache directory
+  - /your/download/path:/books # Downloaded books
+  - /client/path:/client/path # Optional: For Torrent/Usenet downloads, match your client directory exactly. 
 ```
 
-## 📝 Logging
+> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
 
-Logs are available in:
+> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
 
-- Container: `/var/logs/cwa-book-downloader.log`
-- Docker logs: Access via `docker logs`
+## ⚙️ Configuration
 
-## 🤝 Contributing
+### Search Modes
 
-Contributions are welcome! Feel free to submit a Pull Request.
+**Direct** (default)
+- Works out of the box, no setup required
+- Searches a huge library of books directly
+- Returns downloadable releases immediately
 
-## 📄 License
+**Universal**
+- Cleaner search results via metadata providers (Hardcover is recommended)
+- Aggregates releases from multiple configured sources
+- Full Audiobook support
+- Requires manual setup (API keys, additional sources)
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+### Environment Variables
 
-## ⚠️ Important Disclaimers
+Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
 
-### Copyright Notice
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FLASK_PORT` | Web interface port | `8084` |
+| `INGEST_DIR` | Book download directory | `/books` |
+| `TZ` | Container timezone | `UTC` |
+| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` |
+| `SEARCH_MODE` | `direct` or `universal` | `direct` |
+| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
 
-While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for:
+See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
 
-- Ensuring they have the right to download requested materials
-- Respecting copyright laws and intellectual property rights
-- Using the tool in compliance with their local regulations
+Some of the additional options available in Settings:
+- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
+- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
+- **AudiobookBay** - Web scraping source for audiobook torrents (audiobooks only)
+- **IRC** - Add details for IRC book sources and download directly from the UI
+- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
+- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
+- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
+- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
+- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
 
-### Duplicate Downloads Warning
+## 🐳 Docker Variants
 
-Please note that the current version:
+### Standard
+```bash
+docker compose up -d
+```
 
-- Does not check for existing files in the download directory
-- Does not verify if books already exist in your Calibre database
-- Exercise caution when requesting multiple books to avoid duplicates
+The full-featured image with built-in Cloudflare bypass.
 
-## 💬 Support
+#### Enable Tor Routing
+Routes all traffic through Tor for enhanced privacy:
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
+docker compose -f docker-compose.tor.yml up -d
+```
 
-For issues or questions, please file an issue on the GitHub repository.
+**Notes:**
+- Requires `NET_ADMIN` and `NET_RAW` capabilities
+- Timezone is auto-detected from Tor exit node
+- Custom DNS/proxy settings are ignored when Tor is active
 
+### Lite
+A smaller image without the built-in Cloudflare bypasser. Ideal for:
+
+- **External bypassers** - Already running FlareSolverr or ByParr for other services
+- **Fast downloads** - Using fast download sources
+- **Alternative sources only** - Exclusively using Prowlarr, AudiobookBay, IRC, or other sources
+- **Audiobooks** - Using Shelfmark exclusively for audiobooks
+
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
+docker compose -f docker-compose.lite.yml up -d
+```
+
+If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
+
+## 🔐 Authentication
+
+Authentication is optional but recommended for shared or exposed instances. Multiple authentication methods are available in Settings:
+
+**1. Single Username/Password**
+
+**2. Proxy (Forward) Authentication**
+
+Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
+
+**3. OIDC (OpenID Connect)**
+
+Integrate with your identity provider (Authelia, Authentik, Keycloak, etc.) for single sign-on. Supports PKCE flow, auto-discovery, group-based admin mapping, and auto-provisioning of new users.
+
+**4. Calibre-Web Database**
+
+If you're running Calibre-Web, you can reuse its user database by mounting it:
+
+```yaml
+volumes:
+  - /path/to/calibre-web/app.db:/auth/app.db:ro
+```
+
+### Multi-User Support
+
+With any authentication method enabled, Shelfmark supports multi-user management with admin/user roles. Users can have per-user settings for download destinations, email recipients, and notification preferences. Non-admin users only see their own downloads and can submit book requests for admin review. Admins can configure request policies per source to control whether users can download directly, must submit a request, or are blocked entirely.
+
+## Project Scope
+
+Shelfmark is a manual search and download tool, the entry point to your book library, not a library manager. It finds books, downloads them, and sends them to a configured destination. That's the full scope.
+
+Shelfmark intentionally does not:
+
+- **Track or manage your library** - it doesn't know or care what you already own
+- **Integrate with library software** - what happens after delivery is up to your library tool
+- **Monitor authors, series, or new releases** - there is no background automation
+- **Queue future downloads** - if a book isn't available now, Shelfmark won't watch for it
+
+These are non-goals, not missing features.
+
+## Contributing
+
+Shelfmark's core feature set is complete. Development focuses on stability, bug fixes, quality-of-life improvements, and refining the search experience. Contributions in these areas are welcome, please file issues or submit pull requests on GitHub.
+
+Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed. If you're unsure whether something fits, open a discussion first.
+
+## Health Monitoring
+
+The application exposes a health endpoint at `/api/health` (no authentication required). Add a health check to your compose:
+
+```yaml
+healthcheck:
+  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/health"]
+  interval: 30s
+  timeout: 30s
+  retries: 3
+```
+
+## Logging
+
+Logs are available via:
+- `docker logs <container-name>`
+- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
+
+Log level is configurable via Settings or `LOG_LEVEL` environment variable.
+
+## Development
+
+```bash
+# Frontend development
+make install     # Install dependencies
+make dev         # Start Vite dev server (localhost:5173)
+make build       # Production build
+make typecheck   # TypeScript checks
+
+# Backend (Docker)
+make up          # Start backend via docker-compose.dev.yml
+make down        # Stop services
+make refresh     # Rebuild and restart
+make restart     # Restart container
+```
+
+The frontend dev server proxies to the backend on port 8084.
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## ⚠️ Disclaimer
+
+Shelfmark is a search interface that displays results from external metadata providers and sources. It does not host, store, or distribute any content. The developers are not responsible for how the tool is used or what is accessed through it.
+
+Users are solely responsible for:
+- Ensuring they have the legal right to download any material they access
+- Complying with copyright laws and intellectual property rights in their jurisdiction
+- Understanding and accepting the terms of any sources they configure
+
+Use of this tool is entirely at your own risk.
+
+## Support
+
+For issues or questions, please [file an issue](https://github.com/calibrain/shelfmark/issues) on GitHub.

requirements-base.txt

@@ -0,0 +1,19 @@
+flask
+flask-cors
+flask-socketio
+python-socketio
+requests[socks]
+defusedxml
+beautifulsoup4
+tqdm
+dnspython
+gunicorn
+gevent
+gevent-websocket
+psutil
+emoji
+rarfile
+qbittorrent-api
+transmission-rpc
+authlib>=1.6.6,<1.7
+apprise>=1.9.0

requirements-shelfmark.txt

@@ -0,0 +1,4 @@
+pyvirtualdisplay
+pyautogui
+seleniumbase==4.45.10
+python-xlib

requirements.txt

@@ -1,11 +0,0 @@
-flask
-requests[socks]
-beautifulsoup4
-tqdm
-pyvirtualdisplay
-dnspython
-pyautogui
-seleniumbase>=4.41.1
-gunicorn
-python-xlib
-psutil

scripts/fix_permissions.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fix permissions on all configured directories.
+
+This script is called by the entrypoint to ensure all user-configured
+directories have correct ownership. It reads directory paths from:
+- CONFIG_DIR environment variable
+- Config files in CONFIG_DIR/plugins/
+
+Outputs directory paths that need permission fixing (one per line).
+The entrypoint handles the actual chown operations.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def get_directories_from_config() -> set[str]:
+    """Extract all directory paths from config files."""
+    directories = set()
+
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    plugins_dir = config_dir / "plugins"
+
+    if not plugins_dir.exists():
+        return directories
+
+    # Keys that contain directory paths
+    directory_keys = {
+        # Main destinations
+        "DESTINATION",
+        "DESTINATION_AUDIOBOOK",
+        # Content type routing directories
+        "AA_CONTENT_TYPE_DIR_FICTION",
+        "AA_CONTENT_TYPE_DIR_NON_FICTION",
+        "AA_CONTENT_TYPE_DIR_UNKNOWN",
+        "AA_CONTENT_TYPE_DIR_MAGAZINE",
+        "AA_CONTENT_TYPE_DIR_COMIC",
+        "AA_CONTENT_TYPE_DIR_STANDARDS",
+        "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+        "AA_CONTENT_TYPE_DIR_OTHER",
+        # Legacy keys (in case of old configs)
+        "INGEST_DIR",
+        "INGEST_DIR_AUDIOBOOK",
+        "INGEST_DIR_BOOK_FICTION",
+        "INGEST_DIR_BOOK_NON_FICTION",
+        "INGEST_DIR_BOOK_UNKNOWN",
+        "INGEST_DIR_MAGAZINE",
+        "INGEST_DIR_COMIC_BOOK",
+        "INGEST_DIR_STANDARDS_DOCUMENT",
+        "INGEST_DIR_MUSICAL_SCORE",
+        "INGEST_DIR_OTHER",
+        "LIBRARY_PATH",
+        "LIBRARY_PATH_AUDIOBOOK",
+    }
+
+    # Read all JSON config files
+    for config_file in plugins_dir.glob("*.json"):
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+
+            for key in directory_keys:
+                if key in config:
+                    value = config[key]
+                    if value and isinstance(value, str) and value.startswith("/"):
+                        directories.add(value)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    return directories
+
+
+def main():
+    """Output all configured directories that exist."""
+    directories = get_directories_from_config()
+
+    # Filter to directories that actually exist
+    existing = []
+    for dir_path in directories:
+        path = Path(dir_path)
+        if path.exists() and path.is_dir():
+            existing.append(dir_path)
+
+    # Output one directory per line
+    for dir_path in sorted(existing):
+        print(dir_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_env_docs.py

@@ -0,0 +1,440 @@
+#!/usr/bin/env python3
+"""Generate markdown documentation for environment variables from the settings registry.
+
+This script extracts all settings that support environment variable configuration
+and generates a comprehensive markdown file documenting each option.
+
+Usage:
+    python scripts/generate_env_docs.py [--output path/to/output.md]
+
+The generated documentation includes:
+- Environment variable name
+- Description
+- Type (string, number, boolean, etc.)
+- Default value
+- Organizational grouping by settings tab/group
+"""
+
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Add project root to path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+
+def get_field_type_name(field) -> str:
+    """Get a human-readable type name for a field."""
+    from shelfmark.core.settings_registry import (
+        CheckboxField,
+        MultiSelectField,
+        NumberField,
+        OrderableListField,
+        PasswordField,
+        SelectField,
+        TextField,
+    )
+
+    if isinstance(field, CheckboxField):
+        return "boolean"
+    elif isinstance(field, NumberField):
+        return "number"
+    elif isinstance(field, SelectField):
+        return "string (choice)"
+    elif isinstance(field, MultiSelectField):
+        return "string (comma-separated)"
+    elif isinstance(field, OrderableListField):
+        return "JSON array"
+    elif isinstance(field, PasswordField):
+        return "string (secret)"
+    elif isinstance(field, TextField):
+        return "string"
+    else:
+        return "string"
+
+
+def format_default_value(field) -> str:
+    """Format the default value for display."""
+    default = field.default
+
+    if default is None:
+        return "_none_"
+    elif isinstance(default, bool):
+        return f"`{str(default).lower()}`"
+    elif isinstance(default, (int, float)):
+        return f"`{default}`"
+    elif isinstance(default, str):
+        if default == "":
+            return "_empty string_"
+        return f"`{default}`"
+    elif isinstance(default, list):
+        if not default:
+            return "_empty list_"
+        # For simple lists, show comma-separated values
+        if all(isinstance(item, str) for item in default):
+            return f"`{','.join(default)}`"
+        # For complex lists (e.g., OrderableListField defaults), summarize
+        return f"_see UI for defaults_"
+    else:
+        return f"`{default}`"
+
+
+def get_select_options(field) -> Optional[List[str]]:
+    """Get the available options for a SelectField.
+
+    Returns options formatted as 'value (label)' or just 'value' if they match,
+    so users know the actual values to use in environment variables.
+    """
+    from shelfmark.core.settings_registry import SelectField
+
+    if not isinstance(field, SelectField):
+        return None
+
+    options = field.options
+    if callable(options):
+        try:
+            options = options()
+        except Exception:
+            return None
+
+    if not options:
+        return None
+
+    result = []
+    for opt in options:
+        value = opt.get("value", "")
+        label = opt.get("label", "")
+
+        # Format as "value (label)" unless they're the same or value is empty
+        if value == "":
+            result.append(f'`""` ({label})')
+        elif value == label or not label:
+            result.append(f"`{value}`")
+        else:
+            result.append(f"`{value}` ({label})")
+
+    return result
+
+
+def _generate_bootstrap_env_docs() -> List[str]:
+    """Generate documentation for bootstrap environment variables from env.py."""
+    # These are environment variables defined in env.py that are used before
+    # the settings registry is available
+    bootstrap_vars = [
+        {
+            "name": "CONFIG_DIR",
+            "description": "Directory for storing configuration files and plugin settings.",
+            "type": "string (path)",
+            "default": "/config",
+        },
+        {
+            "name": "LOG_ROOT",
+            "description": "Root directory for log files.",
+            "type": "string (path)",
+            "default": "/var/log/",
+        },
+        {
+            "name": "TMP_DIR",
+            "description": "Staging directory for downloads before moving to destination.",
+            "type": "string (path)",
+            "default": "/tmp/shelfmark",
+        },
+        {
+            "name": "ENABLE_LOGGING",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
+            "type": "boolean",
+            "default": "true",
+        },
+        {
+            "name": "FLASK_HOST",
+            "description": "Host address for the Flask web server.",
+            "type": "string",
+            "default": "0.0.0.0",
+        },
+        {
+            "name": "FLASK_PORT",
+            "description": "Port number for the Flask web server.",
+            "type": "number",
+            "default": "8084",
+        },
+        {
+            "name": "SESSION_COOKIE_SECURE",
+            "description": "Enable secure cookies (requires HTTPS).",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "CWA_DB_PATH",
+            "description": "Path to the Calibre-Web database for authentication integration.",
+            "type": "string (path)",
+            "default": "/auth/app.db",
+        },
+        {
+            "name": "DOCKERMODE",
+            "description": "Indicates the application is running inside a Docker container.",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "ONBOARDING",
+            "description": "Show the onboarding wizard on first run. Set to false to skip (useful for ephemeral storage).",
+            "type": "boolean",
+            "default": "true",
+        },
+    ]
+
+    lines = [
+        "## Bootstrap Configuration",
+        "",
+        "These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
+        "",
+        "| Variable | Description | Type | Default |",
+        "|----------|-------------|------|---------|",
+    ]
+
+    for var in bootstrap_vars:
+        lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
+
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for var in bootstrap_vars:
+        lines.append(f"#### `{var['name']}`")
+        lines.append("")
+        lines.append(var["description"])
+        lines.append("")
+        lines.append(f"- **Type:** {var['type']}")
+        lines.append(f"- **Default:** `{var['default']}`")
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def generate_env_docs() -> str:
+    """Generate markdown documentation for all environment variables."""
+    # Import settings modules to ensure all settings are registered
+    import shelfmark.config.settings  # noqa: F401
+    import shelfmark.config.security  # noqa: F401
+    import shelfmark.release_sources.irc.settings  # noqa: F401
+    import shelfmark.release_sources.prowlarr.settings  # noqa: F401
+    import shelfmark.metadata_providers.hardcover  # noqa: F401
+    import shelfmark.metadata_providers.openlibrary  # noqa: F401
+    import shelfmark.metadata_providers.googlebooks  # noqa: F401
+
+    from shelfmark.core.settings_registry import (
+        ActionButton,
+        HeadingField,
+        get_all_groups,
+        get_all_settings_tabs,
+    )
+
+    tabs = get_all_settings_tabs()
+    groups = {g.name: g for g in get_all_groups()}
+
+    # Organize tabs by group
+    grouped_tabs: Dict[Optional[str], List] = {None: []}
+    for group_name in groups:
+        grouped_tabs[group_name] = []
+
+    for tab in tabs:
+        group_name = tab.group
+        if group_name not in grouped_tabs:
+            grouped_tabs[group_name] = []
+        grouped_tabs[group_name].append(tab)
+
+    # Build markdown output
+    lines = [
+        "# Environment Variables",
+        "",
+        "This document lists all configuration options that can be set via environment variables.",
+        "",
+        "> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
+        "",
+        "## Table of Contents",
+        "",
+    ]
+
+    # Generate TOC
+    toc_entries = [
+        "- [Bootstrap Configuration](#bootstrap-configuration)",
+    ]
+
+    # Ungrouped tabs first
+    for tab in grouped_tabs.get(None, []):
+        anchor = tab.display_name.lower().replace(" ", "-")
+        toc_entries.append(f"- [{tab.display_name}](#{anchor})")
+
+    # Then grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if group_tabs:
+            anchor = group.display_name.lower().replace(" ", "-")
+            toc_entries.append(f"- [{group.display_name}](#{anchor})")
+            for tab in group_tabs:
+                sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
+                toc_entries.append(f"  - [{tab.display_name}](#{sub_anchor})")
+
+    lines.extend(toc_entries)
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Add bootstrap environment variables documentation
+    lines.extend(_generate_bootstrap_env_docs())
+
+    # Generate documentation for ungrouped tabs
+    for tab in grouped_tabs.get(None, []):
+        lines.extend(_generate_tab_docs(tab))
+
+    # Generate documentation for grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if not group_tabs:
+            continue
+
+        lines.append(f"## {group.display_name}")
+        lines.append("")
+
+        for tab in group_tabs:
+            lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
+
+    return "\n".join(lines)
+
+
+def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
+    """Generate documentation for a single settings tab."""
+    from shelfmark.core.settings_registry import ActionButton, CustomComponentField, HeadingField
+
+    lines = []
+
+    # Section header
+    if group_prefix:
+        lines.append(f"### {group_prefix}: {tab.display_name}")
+        anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
+    else:
+        lines.append(f"## {tab.display_name}")
+
+    lines.append("")
+
+    # Collect env-supported fields
+    env_fields = []
+    for field in tab.fields:
+        # Skip non-value fields
+        if isinstance(field, (ActionButton, CustomComponentField, HeadingField)):
+            continue
+
+        # Skip fields that don't support ENV vars
+        if not getattr(field, "env_supported", True):
+            continue
+
+        env_fields.append(field)
+
+    if not env_fields:
+        lines.append("_No environment variables for this section._")
+        lines.append("")
+        return lines
+
+    # Generate table
+    lines.append("| Variable | Description | Type | Default |")
+    lines.append("|----------|-------------|------|---------|")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        description = field.description or field.label
+        # Clean up description for table (remove newlines, escape pipes)
+        description = description.replace("\n", " ").replace("|", "\\|").strip()
+
+        field_type = get_field_type_name(field)
+        default = format_default_value(field)
+
+        lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
+
+    lines.append("")
+
+    # Add detailed documentation for each field
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        lines.append(f"#### `{env_var}`")
+        lines.append("")
+        lines.append(f"**{field.label}**")
+        lines.append("")
+
+        if field.description:
+            lines.append(field.description)
+            lines.append("")
+
+        lines.append(f"- **Type:** {get_field_type_name(field)}")
+        lines.append(f"- **Default:** {format_default_value(field)}")
+
+        if getattr(field, "required", False):
+            lines.append("- **Required:** Yes")
+
+        if getattr(field, "requires_restart", False):
+            lines.append("- **Requires restart:** Yes")
+
+        # Show options for SelectField
+        options = get_select_options(field)
+        if options:
+            lines.append(f"- **Options:** {', '.join(options)}")
+
+        # Show constraints for NumberField
+        from shelfmark.core.settings_registry import NumberField
+        if isinstance(field, NumberField):
+            constraints = []
+            if field.min_value is not None:
+                constraints.append(f"min: {field.min_value}")
+            if field.max_value is not None:
+                constraints.append(f"max: {field.max_value}")
+            if constraints:
+                lines.append(f"- **Constraints:** {', '.join(constraints)}")
+
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate markdown documentation for environment variables"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=project_root / "docs" / "environment-variables.md",
+        help="Output file path (default: docs/environment-variables.md)",
+    )
+    parser.add_argument(
+        "--stdout",
+        action="store_true",
+        help="Print to stdout instead of file",
+    )
+    args = parser.parse_args()
+
+    docs = generate_env_docs()
+
+    if args.stdout:
+        print(docs)
+    else:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(docs)
+        print(f"Generated: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_clients.py

@@ -0,0 +1,555 @@
+#!/usr/bin/env python3
+"""
+Test script for download client implementations.
+
+Usage:
+    1. Start the test stack:
+       docker compose -f docker-compose.test-clients.yml up -d
+
+    2. Wait for containers to initialize (first run takes ~30s)
+
+    3. Run this script to verify clients are accessible:
+       python scripts/test_clients.py
+
+    4. Access cwabd at http://localhost:8084
+       - Go to Settings > Prowlarr > Download Clients
+       - Select a client from the dropdown
+       - Click "Test Connection" to verify
+
+Web UIs:
+    - cwabd:        http://localhost:8084
+    - qBittorrent:  http://localhost:8080
+    - Transmission: http://localhost:9091
+    - Deluge:       http://localhost:8112
+    - NZBGet:       http://localhost:6789
+    - SABnzbd:      http://localhost:8085
+    - rTorrent:     http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
+
+Prerequisites (for running this script locally):
+    pip install requests transmission-rpc qbittorrent-api
+
+First-Time Setup:
+    qBittorrent:
+        - Check container logs for temporary password: docker logs test-qbittorrent
+        - Login at http://localhost:8080, change password to something known
+        - Default username is 'admin'
+
+    Transmission:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    Deluge:
+        - Access Web UI at http://localhost:8112 (default password: deluge)
+
+    NZBGet:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    SABnzbd:
+        - Complete the setup wizard at http://localhost:8085
+        - API key will be auto-detected by this script
+        - In cwabd, copy API key from SABnzbd Config > General
+"""
+
+import sys
+import time
+from xmlrpc import client
+
+# Test configuration - matches docker-compose.test-clients.yml
+CONFIG = {
+    # Usenet clients
+    "nzbget": {
+        "url": "http://localhost:6789",
+        "username": "admin",
+        "password": "admin",
+    },
+    "sabnzbd": {
+        "url": "http://localhost:8085",
+        "api_key": None,  # Will be read from config on first run
+    },
+    # Torrent clients
+    "qbittorrent": {
+        "url": "http://localhost:8080",
+        "username": "admin",
+        "password": "5NCngsHXm",  # Temp password from: docker logs test-qbittorrent | grep password
+    },
+    "transmission": {
+        "url": "http://localhost:9091",
+        "username": "admin",
+        "password": "admin",
+    },
+    "deluge": {
+        "url": "http://localhost:8112",
+        "password": "deluge",
+    },
+    "rtorrent": {
+        "url": "http://localhost:8000/RPC2",
+    },
+}
+
+# Test magnet link (Ubuntu ISO - legal, small metadata)
+TEST_MAGNET = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=ubuntu-22.04.3-live-server-amd64.iso"
+
+
+def test_nzbget():
+    """Test NZBGet connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing NZBGet")
+    print("=" * 50)
+
+    url = CONFIG["nzbget"]["url"]
+    username = CONFIG["nzbget"]["username"]
+    password = CONFIG["nzbget"]["password"]
+
+    try:
+        # Test connection via JSON-RPC
+        rpc_url = f"{url}/jsonrpc"
+        response = requests.post(
+            rpc_url,
+            json={"method": "version", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("result", "unknown")
+        print(f"  Connected to NZBGet {version}")
+
+        # Test status
+        response = requests.post(
+            rpc_url,
+            json={"method": "status", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        status = response.json().get("result", {})
+        print(f"  Server state: {'Paused' if status.get('ServerPaused') else 'Running'}")
+        print(f"  Downloads in queue: {status.get('DownloadedSizeMB', 0)} MB downloaded")
+
+        print("  SUCCESS: NZBGet is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to NZBGet")
+        print("  Is the container running? docker ps | grep nzbget")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_sabnzbd():
+    """Test SABnzbd connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing SABnzbd")
+    print("=" * 50)
+
+    url = CONFIG["sabnzbd"]["url"]
+    api_key = CONFIG["sabnzbd"]["api_key"]
+
+    # Try to get API key from config if not set
+    if not api_key:
+        try:
+            import os
+            ini_path = ".local/test-clients/sabnzbd/config/sabnzbd.ini"
+            if os.path.exists(ini_path):
+                with open(ini_path) as f:
+                    for line in f:
+                        if line.startswith("api_key"):
+                            api_key = line.split("=")[1].strip()
+                            print(f"  Found API key in config: {api_key[:8]}...")
+                            break
+        except Exception as e:
+            print(f"  Could not read API key from config: {e}")
+
+    if not api_key:
+        print("  ERROR: No API key configured")
+        print("  Please access http://localhost:8085 and complete initial setup")
+        print("  Then copy the API key from Config > General")
+        return False
+
+    try:
+        # Test connection
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "version", "output": "json"},
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("version", "unknown")
+        print(f"  Connected to SABnzbd {version}")
+
+        # Test queue status
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "queue", "output": "json"},
+            timeout=10,
+        )
+        queue = response.json().get("queue", {})
+        print(f"  Queue status: {queue.get('status', 'unknown')}")
+        print(f"  Items in queue: {len(queue.get('slots', []))}")
+
+        print("  SUCCESS: SABnzbd is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to SABnzbd")
+        print("  Is the container running? docker ps | grep sabnzbd")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_qbittorrent():
+    """Test qBittorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing qBittorrent")
+    print("=" * 50)
+
+    try:
+        import qbittorrentapi
+
+        url = CONFIG["qbittorrent"]["url"]
+        username = CONFIG["qbittorrent"]["username"]
+        password = CONFIG["qbittorrent"]["password"]
+
+        # Parse URL for host/port
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+
+        client = qbittorrentapi.Client(
+            host=parsed.hostname,
+            port=parsed.port or 8080,
+            username=username,
+            password=password,
+        )
+
+        # Test connection
+        client.auth_log_in()
+        version = client.app.version
+        print(f"  Connected to qBittorrent {version}")
+
+        # Get torrent list
+        torrents = client.torrents_info()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        result = client.torrents_add(urls=TEST_MAGNET, is_paused=True)
+        if result == "Ok.":
+            # Wait a moment for it to be added
+            time.sleep(1)
+            torrents = client.torrents_info()
+            if torrents:
+                test_torrent = torrents[-1]  # Most recently added
+                print(f"  Added test torrent: {test_torrent.name[:50]}...")
+                print(f"  Status: {test_torrent.state}")
+
+                # Remove it
+                client.torrents_delete(torrent_hashes=test_torrent.hash, delete_files=True)
+                print("  Removed test torrent")
+        else:
+            print(f"  Add result: {result}")
+
+        print("  SUCCESS: qBittorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: qbittorrent-api not installed")
+        print("  Run: pip install qbittorrent-api")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Forbidden" in str(e) or "401" in str(e):
+            print("\n  Authentication failed. Check password:")
+            print("  1. docker logs test-qbittorrent | grep password")
+            print("  2. Login to http://localhost:8080 and set a known password")
+        return False
+
+
+def test_transmission():
+    """Test Transmission connection."""
+    print("\n" + "=" * 50)
+    print("Testing Transmission")
+    print("=" * 50)
+
+    try:
+        from transmission_rpc import Client
+        from urllib.parse import urlparse
+
+        url = CONFIG["transmission"]["url"]
+        parsed = urlparse(url)
+
+        client = Client(
+            host=parsed.hostname,
+            port=parsed.port or 9091,
+            username=CONFIG["transmission"]["username"],
+            password=CONFIG["transmission"]["password"],
+        )
+
+        # Test connection
+        session = client.get_session()
+        print(f"  Connected to Transmission {session.version}")
+
+        # Get torrent list
+        torrents = client.get_torrents()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent = client.add_torrent(TEST_MAGNET, paused=True)
+        print(f"  Added test torrent: {torrent.name[:50]}...")
+
+        # Get status
+        status = client.get_torrent(torrent.id)
+        print(f"  Status: {status.status} ({status.percent_done * 100:.1f}%)")
+
+        # Remove it
+        client.remove_torrent(torrent.id, delete_data=True)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: Transmission is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: transmission-rpc not installed")
+        print("  Run: pip install transmission-rpc")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_deluge():
+    """Test Deluge Web UI (JSON-RPC) connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing Deluge")
+    print("=" * 50)
+
+    base_url = CONFIG["deluge"]["url"].rstrip("/")
+    password = CONFIG["deluge"]["password"]
+    rpc_url = f"{base_url}/json"
+
+    def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
+        payload = {"id": rpc_id, "method": method, "params": list(params)}
+        resp = session.post(rpc_url, json=payload, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("error"):
+            err = data["error"]
+            if isinstance(err, dict):
+                raise Exception(err.get("message") or str(err))
+            raise Exception(str(err))
+        return data.get("result")
+
+    try:
+        session = requests.Session()
+
+        # Authenticate to Deluge Web
+        if rpc_call(session, 1, "auth.login", password) is not True:
+            raise Exception("Authentication failed (check Deluge Web UI password)")
+
+        # Ensure Deluge Web is connected to a daemon
+        if rpc_call(session, 2, "web.connected") is not True:
+            hosts = rpc_call(session, 3, "web.get_hosts") or []
+            if not hosts:
+                raise Exception(
+                    "Deluge Web UI isn't connected to Deluge core (no hosts configured). "
+                    "Add/connect a daemon in Deluge Web UI → Connection Manager."
+                )
+
+            host_id = hosts[0][0]
+            for entry in hosts:
+                if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
+                    host_id = entry[0]
+                    break
+
+            rpc_call(session, 4, "web.connect", host_id)
+
+            if rpc_call(session, 5, "web.connected") is not True:
+                raise Exception(
+                    "Deluge Web UI couldn't connect to Deluge core. "
+                    "Check Deluge Web UI → Connection Manager."
+                )
+
+        version = rpc_call(session, 6, "daemon.info")
+        print(f"  Connected to Deluge {version}")
+
+        torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
+
+        if torrent_id:
+            torrent_id = str(torrent_id)
+            print(f"  Added test torrent: {torrent_id[:20]}...")
+
+            status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
+            state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
+            progress = status.get("progress", 0) if isinstance(status, dict) else 0
+            print(f"  Status: {state} ({progress:.1f}%)")
+
+            rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
+            print("  Removed test torrent")
+        else:
+            print("  WARNING: Could not add test torrent")
+
+        print("  SUCCESS: Deluge is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to Deluge Web UI")
+        print("  Is the container running? docker ps | grep deluge")
+        return False
+    except requests.exceptions.Timeout:
+        print("  ERROR: Deluge Web UI connection timed out")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "auth" in str(e).lower() or "login" in str(e).lower():
+            print("  Check Deluge Web UI password (default: deluge)")
+        return False
+
+def test_rtorrent():
+    """Test rTorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing rTorrent")
+    print("=" * 50)
+
+    try:
+        import xmlrpc.client
+
+        url = "http://localhost:8000/RPC2"
+        client = xmlrpc.client.ServerProxy(url)
+
+        # Test connection
+        version = client.system.library_version()
+        print(f"  Connected to rTorrent {version}")
+
+        # default download directory test
+        default_dir = client.directory.default()
+        print(f"  Default download directory: {default_dir}")
+
+        # Get torrent list
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+
+        label = "automated"
+
+        commands = []
+        if label:
+            commands.append(f"d.custom1.set={label}")
+
+        download_dir = "/downloads"
+        if download_dir:
+            commands.append(f"d.directory_base.set={download_dir}")
+
+        # rtorrent is weird in that it doesn't return the torrent ID/hash on add
+        client.load.start("", TEST_MAGNET, ";".join(commands))
+        
+        # but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
+        torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
+        print(f"  Added test torrent: {torrent_id}")
+
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")        
+
+        torrent_list = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal={{d.hash=,cat={torrent_id}}}"
+            "d.hash=",
+            "d.state=",
+            "d.completed_bytes=",
+            "d.size_bytes=",
+            "d.down.rate=",
+            "d.up.rate=",
+            "d.custom1=",
+            "d.complete=",
+        )
+        torrent = torrent_list[0]
+
+        if not torrent:
+            print("  ERROR: Could not find added torrent in list")
+            return False
+        
+        # let's test the base path call
+        details = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal=d.hash=,cat={torrent_id}",
+            "d.base_path=",
+        )
+
+        base_path = details[0][0] if details else None
+
+        print(f"  Base path: {base_path}")
+        client.d.erase(torrent_id)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: rTorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: xmlrpc.client not available")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Connection refused" in str(e):
+            print("  Is the container running? docker ps | grep rtorrent")
+        return False
+
+
+def main():
+    print("Download Client Test Suite")
+    print("=" * 50)
+    print("Make sure containers are running:")
+    print("  docker compose -f docker-compose.test-clients.yml up -d")
+
+    results = {}
+
+    # Test usenet clients
+    print("\n" + "=" * 50)
+    print("USENET CLIENTS")
+    print("=" * 50)
+    results["nzbget"] = test_nzbget()
+    results["sabnzbd"] = test_sabnzbd()
+
+    # Test torrent clients
+    print("\n" + "=" * 50)
+    print("TORRENT CLIENTS")
+    print("=" * 50)
+    results["qbittorrent"] = test_qbittorrent()
+    results["transmission"] = test_transmission()
+    results["deluge"] = test_deluge()
+    results["rtorrent"] = test_rtorrent()
+
+    # Summary
+    print("\n" + "=" * 50)
+    print("SUMMARY")
+    print("=" * 50)
+
+    for client, success in results.items():
+        status = "PASS" if success else "FAIL"
+        print(f"  {client}: {status}")
+
+    passed = sum(results.values())
+    total = len(results)
+    print(f"\n  Total: {passed}/{total} passed")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

shelfmark/__init__.py

@@ -0,0 +1 @@
+"""Shelfmark - book search and download service."""

shelfmark/__main__.py

@@ -0,0 +1,8 @@
+"""Package entry point for `python -m shelfmark`."""
+
+from shelfmark.main import app, socketio
+from shelfmark.config.env import FLASK_HOST, FLASK_PORT
+from shelfmark.core.config import config
+
+if __name__ == "__main__":
+    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

shelfmark/api/__init__.py

@@ -0,0 +1 @@
+"""API module - WebSocket handling."""

shelfmark/api/websocket.py

@@ -0,0 +1,251 @@
+"""WebSocket manager for real-time status updates."""
+
+import logging
+import threading
+from typing import Optional, Dict, Any, Callable, List
+
+from flask_socketio import SocketIO, join_room, leave_room
+
+logger = logging.getLogger(__name__)
+
+
+class WebSocketManager:
+    """Manages WebSocket connections and broadcasts."""
+
+    def __init__(self):
+        self.socketio: Optional[SocketIO] = None
+        self._enabled = False
+        self._connection_count = 0
+        self._connection_lock = threading.Lock()
+        self._on_first_connect_callbacks: List[Callable[[], None]] = []
+        self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
+        self._needs_rewarm = False  # Flag to trigger warmup callbacks on next connect
+        self._user_rooms: Dict[str, int] = {}  # room_name -> ref count
+        self._sid_rooms: Dict[str, str] = {}  # sid -> room_name
+        self._rooms_lock = threading.Lock()
+        self._queue_status_fn: Optional[Callable] = None  # Reference to queue_status()
+
+    def init_app(self, app, socketio: SocketIO):
+        """Initialize the WebSocket manager with Flask-SocketIO instance."""
+        self.socketio = socketio
+        self._enabled = True
+        logger.info("WebSocket manager initialized")
+
+    def register_on_first_connect(self, callback: Callable[[], None]):
+        """Register a callback for when the first client connects."""
+        self._on_first_connect_callbacks.append(callback)
+        logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
+
+    def register_on_all_disconnect(self, callback: Callable[[], None]):
+        """Register a callback for when all clients disconnect."""
+        self._on_all_disconnect_callbacks.append(callback)
+        logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
+
+    def request_warmup_on_next_connect(self):
+        """Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
+        with self._connection_lock:
+            self._needs_rewarm = True
+            logger.debug("Warmup requested for next client connect")
+
+    def client_connected(self):
+        """Track a new client connection. Call this from the connect event handler."""
+        with self._connection_lock:
+            was_zero = self._connection_count == 0
+            needs_rewarm = self._needs_rewarm
+            self._connection_count += 1
+            current_count = self._connection_count
+            # Clear rewarm flag if we're going to trigger warmup
+            if was_zero or needs_rewarm:
+                self._needs_rewarm = False
+
+        logger.debug(f"Client connected. Active connections: {current_count}")
+
+        # Trigger warmup callbacks if this is the first connection OR if rewarm was requested
+        # (rewarm is requested when bypasser shuts down due to idle while clients are connected)
+        if was_zero or needs_rewarm:
+            reason = "First client connected" if was_zero else "Rewarm requested after idle shutdown"
+            logger.info(f"{reason}, triggering warmup callbacks...")
+            for callback in self._on_first_connect_callbacks:
+                try:
+                    # Run callbacks in a separate thread to not block the connection
+                    thread = threading.Thread(target=callback, daemon=True)
+                    thread.start()
+                except Exception as e:
+                    logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}")
+
+    def client_disconnected(self):
+        """Track a client disconnection. Call this from the disconnect event handler."""
+        with self._connection_lock:
+            self._connection_count = max(0, self._connection_count - 1)
+            current_count = self._connection_count
+            is_now_zero = current_count == 0
+
+        logger.debug(f"Client disconnected. Active connections: {current_count}")
+
+        # If all clients have disconnected, trigger cleanup callbacks
+        if is_now_zero:
+            logger.info("All clients disconnected, triggering disconnect callbacks...")
+            for callback in self._on_all_disconnect_callbacks:
+                try:
+                    callback()
+                except Exception as e:
+                    logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}")
+
+    def get_connection_count(self) -> int:
+        """Get the current number of active WebSocket connections."""
+        with self._connection_lock:
+            return self._connection_count
+
+    def has_active_connections(self) -> bool:
+        """Check if there are any active WebSocket connections."""
+        return self.get_connection_count() > 0
+
+    def is_enabled(self) -> bool:
+        """Check if WebSocket is enabled and ready."""
+        return self._enabled and self.socketio is not None
+
+    def set_queue_status_fn(self, fn: Callable):
+        """Set the queue_status function reference for per-room filtering."""
+        self._queue_status_fn = fn
+
+    def _increment_user_room_locked(self, room: str):
+        self._user_rooms[room] = self._user_rooms.get(room, 0) + 1
+
+    def _decrement_user_room_locked(self, room: str):
+        count = self._user_rooms.get(room, 1) - 1
+        if count <= 0:
+            self._user_rooms.pop(room, None)
+        else:
+            self._user_rooms[room] = count
+
+    def _set_sid_room_locked(self, sid: str, room: Optional[str]):
+        current_room = self._sid_rooms.get(sid)
+        if current_room == room:
+            return
+
+        if current_room is not None:
+            leave_room(current_room, sid=sid)
+            if current_room.startswith("user_"):
+                self._decrement_user_room_locked(current_room)
+            self._sid_rooms.pop(sid, None)
+
+        if room is not None:
+            join_room(room, sid=sid)
+            self._sid_rooms[sid] = room
+            if room.startswith("user_"):
+                self._increment_user_room_locked(room)
+
+    def sync_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
+        """Ensure a SID is in exactly one room matching the current session scope."""
+        room: Optional[str] = None
+        if is_admin:
+            room = "admins"
+        elif db_user_id is not None:
+            room = f"user_{db_user_id}"
+
+        with self._rooms_lock:
+            self._set_sid_room_locked(sid, room)
+
+    def join_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
+        """Join the appropriate room based on user role."""
+        self.sync_user_room(sid, is_admin, db_user_id)
+
+    def leave_user_room(self, sid: str, is_admin: bool = False, db_user_id: Optional[int] = None):
+        """Leave whichever room the SID currently belongs to."""
+        del is_admin, db_user_id  # Backward-compatible signature; routing is SID-based.
+        with self._rooms_lock:
+            self._set_sid_room_locked(sid, None)
+
+    def broadcast_status_update(self, status_data: Dict[str, Any]):
+        """Broadcast status update to all connected clients, filtered by user room."""
+        if not self.is_enabled():
+            return
+
+        try:
+            # Admins (and no-auth users) get full status
+            self.socketio.emit('status_update', status_data, to="admins")
+
+            # Each user room gets filtered status
+            with self._rooms_lock:
+                active_rooms = list(self._user_rooms.keys())
+
+            if active_rooms and self._queue_status_fn:
+                for room in active_rooms:
+                    try:
+                        # Extract user_id from room name "user_123"
+                        uid = int(room.split("_", 1)[1])
+                        filtered = self._queue_status_fn(user_id=uid)
+                        self.socketio.emit('status_update', filtered, to=room)
+                    except Exception as e:
+                        logger.error(f"Failed to send status update for room {room}: {e}")
+
+            logger.debug("Broadcasted status update to all rooms")
+        except Exception as e:
+            logger.error(f"Error broadcasting status update: {e}")
+
+    def broadcast_download_progress(self, book_id: str, progress: float, status: str, user_id: Optional[int] = None):
+        """Broadcast download progress update for a specific book."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'book_id': book_id,
+                'progress': progress,
+                'status': status
+            }
+            # Admins always see all progress
+            self.socketio.emit('download_progress', data, to="admins")
+            # If task belongs to a specific user, send to their room too
+            if user_id is not None:
+                room = f"user_{user_id}"
+                with self._rooms_lock:
+                    if room in self._user_rooms:
+                        self.socketio.emit('download_progress', data, to=room)
+            logger.debug(f"Broadcasted progress for book {book_id}: {progress}%")
+        except Exception as e:
+            logger.error(f"Error broadcasting download progress: {e}")
+
+    def broadcast_notification(self, message: str, notification_type: str = 'info'):
+        """Broadcast a notification message to all clients."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'message': message,
+                'type': notification_type
+            }
+            # When calling socketio.emit() outside event handlers, it broadcasts by default
+            self.socketio.emit('notification', data)
+            logger.debug(f"Broadcasted notification: {message}")
+        except Exception as e:
+            logger.error(f"Error broadcasting notification: {e}")
+
+    def broadcast_search_status(
+        self,
+        source: str,
+        provider: str,
+        book_id: str,
+        message: str,
+        phase: str = 'searching'
+    ):
+        """Broadcast search status update for a release source search."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'source': source,
+                'provider': provider,
+                'book_id': book_id,
+                'message': message,
+                'phase': phase,
+            }
+            self.socketio.emit('search_status', data)
+        except Exception as e:
+            logger.error(f"Error broadcasting search status: {e}")
+
+
+# Global WebSocket manager instance
+ws_manager = WebSocketManager()

shelfmark/bypass/__init__.py

@@ -0,0 +1,5 @@
+"""Cloudflare bypass utilities."""
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""

shelfmark/bypass/external_bypasser.py

@@ -0,0 +1,130 @@
+"""External Cloudflare bypasser using FlareSolverr."""
+
+import random
+import time
+from threading import Event
+from typing import TYPE_CHECKING, Optional
+
+import requests
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.utils import normalize_http_url
+from shelfmark.download.network import get_ssl_verify
+
+if TYPE_CHECKING:
+    from shelfmark.download import network
+
+logger = setup_logger(__name__)
+
+# Timeout constants (seconds)
+CONNECT_TIMEOUT = 10
+MAX_READ_TIMEOUT = 120
+READ_TIMEOUT_BUFFER = 15
+
+# Retry settings
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
+
+
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service. Returns HTML or None."""
+    raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
+    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
+    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
+
+    bypasser_url = normalize_http_url(raw_bypasser_url)
+    if not bypasser_url or not bypasser_path:
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
+        return None
+
+    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+
+    try:
+        response = requests.post(
+            f"{bypasser_url}{bypasser_path}",
+            headers={"Content-Type": "application/json"},
+            json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
+            timeout=(CONNECT_TIMEOUT, read_timeout),
+            verify=get_ssl_verify(bypasser_url),
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+
+        solution = result.get('solution')
+        html = solution.get('response', '') if solution else ''
+
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+
+        return html
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+
+    return None
+
+
+def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
+    """Check if operation was cancelled and raise exception if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(f"External bypasser cancelled {context}")
+        raise BypassCancelledException("Bypass cancelled")
+
+
+def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
+    """Sleep for the specified duration, checking for cancellation each second."""
+    for _ in range(int(seconds)):
+        _check_cancelled(cancel_flag, "during backoff")
+        time.sleep(1)
+    remaining = seconds - int(seconds)
+    if remaining > 0:
+        time.sleep(remaining)
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional["network.AAMirrorSelector"] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML via external bypasser with retries and mirror rotation."""
+    from shelfmark.download import network as network_module
+
+    sel = selector or network_module.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        _check_cancelled(cancel_flag, "by user")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        _sleep_with_cancellation(delay, cancel_flag)
+
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

shelfmark/bypass/fingerprint.py

@@ -0,0 +1,57 @@
+"""Browser fingerprint profile management for bypass stealth."""
+
+import random
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+COMMON_RESOLUTIONS = [
+    (1920, 1080, 0.35),  
+    (1366, 768, 0.18),   
+    (1536, 864, 0.10),   
+    (1440, 900, 0.08),   
+    (1280, 720, 0.07),   
+    (1600, 900, 0.06),   
+    (1280, 800, 0.05),   
+    (2560, 1440, 0.04),  
+    (1680, 1050, 0.04),  
+    (1920, 1200, 0.03),  
+]
+
+# Current screen size (module-level singleton)
+_current_screen_size: Optional[tuple[int, int]] = None
+
+
+def get_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    if _current_screen_size is None:
+        _current_screen_size = _generate_screen_size()
+        logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
+    return _current_screen_size
+
+
+def rotate_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    old_size = _current_screen_size
+    _current_screen_size = _generate_screen_size()
+    width, height = _current_screen_size
+
+    if old_size:
+        logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
+    else:
+        logger.info(f"Generated screen size: {width}x{height}")
+
+    return _current_screen_size
+
+
+def clear_screen_size() -> None:
+    global _current_screen_size
+    _current_screen_size = None
+
+
+def _generate_screen_size() -> tuple[int, int]:
+    resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
+    weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
+    return random.choices(resolutions, weights=weights)[0]

shelfmark/bypass/internal_bypasser.py

@@ -0,0 +1,974 @@
+import asyncio
+import os
+import random
+import signal
+import socket
+import subprocess
+import threading
+import time
+import traceback
+from datetime import datetime
+from threading import Event
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+import requests
+from seleniumbase import cdp_driver
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.bypass.fingerprint import get_screen_size
+from shelfmark.config import env
+from shelfmark.config.env import LOG_DIR
+from shelfmark.config.settings import RECORDING_DIR
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download import network
+from shelfmark.download.network import get_proxies, get_ssl_verify
+
+logger = setup_logger(__name__)
+
+# Challenge detection indicators
+CLOUDFLARE_INDICATORS = [
+    "just a moment",
+    "verify you are human",
+    "verifying you are human",
+    "cloudflare.com/products/turnstile",
+]
+
+DDOS_GUARD_INDICATORS = [
+    "ddos-guard",
+    "ddos guard",
+    "checking your browser before accessing",
+    "complete the manual check to continue",
+    "could not verify your browser automatically",
+]
+
+DISPLAY = {
+    "ffmpeg": None,
+    "ffmpeg_output": None,
+}
+LOCKED = threading.Lock()
+
+
+class _CdpWorker:
+    def __init__(self) -> None:
+        self._thread: Optional[threading.Thread] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._ready = threading.Event()
+        self._lock = threading.Lock()
+
+    def _run(self) -> None:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        self._loop = loop
+        self._ready.set()
+        loop.run_forever()
+        try:
+            pending = asyncio.all_tasks(loop)
+            for task in pending:
+                task.cancel()
+            if pending:
+                loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+        except Exception:
+            pass
+        finally:
+            loop.close()
+
+    def start(self) -> None:
+        with self._lock:
+            if self._thread and self._thread.is_alive():
+                return
+            self._ready.clear()
+            self._thread = threading.Thread(
+                target=self._run,
+                name="cdp-worker",
+                daemon=True,
+            )
+            self._thread.start()
+        if not self._ready.wait(timeout=10):
+            raise RuntimeError("CDP worker loop failed to start")
+
+    def run(self, coro: Any, timeout: Optional[float] = None) -> Any:
+        self.start()
+        if not self._loop or self._loop.is_closed():
+            raise RuntimeError("CDP worker loop not available")
+        future = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        return future.result(timeout=timeout)
+
+
+_CDP_WORKER = _CdpWorker()
+
+# Cookie storage - shared with requests library for Cloudflare bypass
+# Structure: {domain: {cookie_name: {value, expiry, ...}}}
+_cf_cookies: dict[str, dict] = {}
+_cf_cookies_lock = threading.Lock()
+
+# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
+_cf_user_agents: dict[str, str] = {}
+
+# Protection cookie names we care about (Cloudflare and DDoS-Guard)
+CF_COOKIE_NAMES = {'cf_clearance', '__cf_bm', 'cf_chl_2', 'cf_chl_prog'}
+DDG_COOKIE_NAMES = {'__ddg1_', '__ddg2_', '__ddg5_', '__ddg8_', '__ddg9_', '__ddg10_', '__ddgid_', '__ddgmark_', 'ddg_last_challenge'}
+
+# Domains requiring full session cookies (not just protection cookies)
+FULL_COOKIE_DOMAINS = {'z-lib.fm', 'z-lib.gs', 'z-lib.id', 'z-library.sk', 'zlibrary-global.se'}
+
+
+def _get_base_domain(domain: str) -> str:
+    """Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
+    return '.'.join(domain.split('.')[-2:]) if '.' in domain else domain
+
+
+def _should_extract_cookie(name: str, extract_all: bool) -> bool:
+    """Determine if a cookie should be extracted based on its name."""
+    if extract_all:
+        return True
+    is_cf = name in CF_COOKIE_NAMES or name.startswith('cf_')
+    is_ddg = name in DDG_COOKIE_NAMES or name.startswith('__ddg')
+    return is_cf or is_ddg
+
+
+def _store_extracted_cookies(
+    *,
+    url: str,
+    cookies: list[Any],
+    user_agent: Optional[str] = None,
+) -> None:
+    """Store filtered bypass cookies (and optional UA) for a URL domain."""
+    parsed = urlparse(url)
+    domain = parsed.hostname or ""
+    if not domain:
+        return
+
+    base_domain = _get_base_domain(domain)
+    extract_all = base_domain in FULL_COOKIE_DOMAINS
+
+    cookies_found: dict[str, dict[str, Any]] = {}
+    for cookie in cookies:
+        name = getattr(cookie, "name", "") or ""
+        if not _should_extract_cookie(name, extract_all):
+            continue
+        expires = getattr(cookie, "expires", None)
+        if expires is not None and expires <= 0:
+            expires = None
+        cookies_found[name] = {
+            "value": getattr(cookie, "value", ""),
+            "domain": getattr(cookie, "domain", None) or domain,
+            "path": getattr(cookie, "path", None) or "/",
+            "expiry": expires,
+            "secure": bool(getattr(cookie, "secure", True)),
+            "httpOnly": True,
+        }
+
+    if not cookies_found:
+        return
+
+    with _cf_cookies_lock:
+        _cf_cookies[base_domain] = cookies_found
+        if user_agent:
+            _cf_user_agents[base_domain] = user_agent
+            logger.debug(f"Stored UA for {base_domain}: {str(user_agent)[:60]}...")
+        else:
+            logger.debug(f"No UA captured for {base_domain}")
+
+    cookie_type = "all" if extract_all else "protection"
+    logger.debug(f"Extracted {len(cookies_found)} {cookie_type} cookies for {base_domain}")
+
+
+async def _extract_cookies_from_cdp(driver, page, url: str) -> None:
+    """Extract cookies from a CDP browser after successful bypass."""
+    try:
+        try:
+            all_cookies = await driver.cookies.get_all(requests_cookie_format=True)
+        except Exception as e:
+            logger.debug(f"Failed to get cookies via CDP: {e}")
+            return
+
+        try:
+            user_agent = await page.evaluate("navigator.userAgent")
+        except Exception:
+            user_agent = None
+
+        _store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
+
+    except Exception as e:
+        logger.debug(f"Failed to extract cookies: {e}")
+
+def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
+    """Get stored cookies for a domain. Returns empty dict if none available."""
+    if not domain:
+        return {}
+
+    base_domain = _get_base_domain(domain)
+
+    with _cf_cookies_lock:
+        cookies = _cf_cookies.get(base_domain, {})
+        if not cookies:
+            return {}
+
+        cf_clearance = cookies.get('cf_clearance', {})
+        if cf_clearance:
+            expiry = cf_clearance.get('expiry')
+            if expiry is None:
+                expiry = cf_clearance.get('expires')
+            if expiry and expiry > 0 and time.time() > expiry:
+                logger.debug(f"CF cookies expired for {base_domain}")
+                _cf_cookies.pop(base_domain, None)
+                return {}
+
+        return {name: c['value'] for name, c in cookies.items()}
+
+
+def has_valid_cf_cookies(domain: str) -> bool:
+    """Check if we have valid Cloudflare cookies for a domain."""
+    return bool(get_cf_cookies_for_domain(domain))
+
+
+def get_cf_user_agent_for_domain(domain: str) -> Optional[str]:
+    """Get the User-Agent that was used during bypass for a domain."""
+    if not domain:
+        return None
+    with _cf_cookies_lock:
+        return _cf_user_agents.get(_get_base_domain(domain))
+
+
+def clear_cf_cookies(domain: str = None) -> None:
+    """Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
+    with _cf_cookies_lock:
+        if domain:
+            base_domain = _get_base_domain(domain)
+            _cf_cookies.pop(base_domain, None)
+            _cf_user_agents.pop(base_domain, None)
+        else:
+            _cf_cookies.clear()
+            _cf_user_agents.clear()
+
+
+def _cleanup_orphan_processes() -> int:
+    """Kill orphan Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode."""
+    if not env.DOCKERMODE:
+        return 0
+
+    _stop_ffmpeg_recording()
+
+    processes_to_kill = ["chrome", "chromium", "Xvfb", "ffmpeg"]
+    total_killed = 0
+
+    logger.debug("Checking for orphan processes...")
+    logger.log_resource_usage()
+
+    for proc_name in processes_to_kill:
+        try:
+            result = subprocess.run(
+                ["pgrep", "-f", proc_name],
+                capture_output=True,
+                text=True,
+                timeout=5
+            )
+            if result.returncode != 0 or not result.stdout.strip():
+                continue
+
+            pids = result.stdout.strip().split('\n')
+            count = len(pids)
+            logger.info(f"Found {count} orphan {proc_name} process(es), killing...")
+
+            kill_result = subprocess.run(
+                ["pkill", "-9", "-f", proc_name],
+                capture_output=True,
+                timeout=5
+            )
+            if kill_result.returncode == 0:
+                total_killed += count
+            else:
+                logger.warning(f"pkill for {proc_name} returned {kill_result.returncode}")
+
+        except subprocess.TimeoutExpired:
+            logger.warning(f"Timeout while checking for {proc_name} processes")
+        except Exception as e:
+            logger.debug(f"Error checking for {proc_name} processes: {e}")
+
+    if total_killed > 0:
+        time.sleep(1)
+        logger.info(f"Cleaned up {total_killed} orphan process(es)")
+        logger.log_resource_usage()
+    else:
+        logger.debug("No orphan processes found")
+
+    return total_killed
+
+async def _get_page_info(page) -> tuple[str, str, str]:
+    """Extract page title, body text, and current URL safely."""
+    try:
+        title = (await page.get_title() or "").lower()
+    except Exception:
+        title = ""
+    try:
+        body = await page.evaluate("document.body ? document.body.innerText : ''")
+        body = (body or "").lower()
+    except Exception:
+        body = ""
+    try:
+        current_url = await page.get_current_url() or ""
+    except Exception:
+        current_url = ""
+    return title, body, current_url
+
+
+def _check_indicators(title: str, body: str, indicators: list[str]) -> Optional[str]:
+    """Check if any indicator is present in title or body. Returns the found indicator or None."""
+    for indicator in indicators:
+        if indicator in title or indicator in body:
+            return indicator
+    return None
+
+def _has_cloudflare_patterns(body: str, url: str) -> bool:
+    """Check for Cloudflare-specific patterns in body or URL."""
+    return "cf-" in body or "cloudflare" in url.lower() or "/cdn-cgi/" in url
+
+async def _detect_challenge_type(page) -> str:
+    """Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'."""
+    try:
+        title, body, current_url = await _get_page_info(page)
+
+        # DDOS-Guard indicators
+        if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS):
+            logger.debug(f"DDOS-Guard indicator found: '{found}'")
+            return "ddos_guard"
+
+        # Cloudflare indicators
+        if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS):
+            logger.debug(f"Cloudflare indicator found: '{found}'")
+            return "cloudflare"
+
+        # Check URL patterns
+        if _has_cloudflare_patterns(body, current_url):
+            return "cloudflare"
+
+        return "none"
+    except Exception as e:
+        logger.warning(f"Error detecting challenge type: {e}")
+        return "none"
+
+async def _is_bypassed(page, escape_emojis: bool = True) -> bool:
+    """Check if the protection has been bypassed."""
+    try:
+        title, body, current_url = await _get_page_info(page)
+        body_len = len(body.strip())
+
+        # Long page content = probably bypassed
+        if body_len > 100000:
+            logger.debug(f"Page content too long, probably bypassed (len: {body_len})")
+            return True
+
+        # Multiple emojis = probably real content
+        if escape_emojis:
+            import emoji
+            if len(emoji.emoji_list(body)) >= 3:
+                logger.debug("Detected emojis in page, probably bypassed")
+                return True
+
+        # Check for protection indicators (means NOT bypassed)
+        if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS):
+            return False
+
+        # Cloudflare URL patterns
+        if _has_cloudflare_patterns(body, current_url):
+            logger.debug("Cloudflare patterns detected in page")
+            return False
+
+        # Page too short = still loading
+        if body_len < 50:
+            logger.debug("Page content too short, might still be loading")
+            return False
+
+        logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {body_len}")
+        return True
+
+    except Exception as e:
+        logger.warning(f"Error checking bypass status: {e}")
+        return False
+
+async def _bypass_method_humanlike(page) -> bool:
+    """Human-like behavior with scroll, wait, and reload."""
+    try:
+        logger.debug("Attempting bypass: human-like interaction")
+        await asyncio.sleep(random.uniform(6, 10))
+
+        try:
+            await page.evaluate("window.scrollTo(0, 10000);")
+            await page.wait()
+            await asyncio.sleep(random.uniform(1, 2))
+            await page.evaluate("window.scrollTo(0, 0);")
+            await page.wait()
+            await asyncio.sleep(random.uniform(2, 3))
+        except Exception as e:
+            logger.debug(f"Scroll behavior failed: {e}")
+
+        if await _is_bypassed(page):
+            return True
+
+        logger.debug("Trying page refresh...")
+        await page.reload(ignore_cache=True)
+        await asyncio.sleep(random.uniform(5, 8))
+
+        if await _is_bypassed(page):
+            return True
+
+        try:
+            await page.solve_captcha()
+            await asyncio.sleep(random.uniform(3, 5))
+        except Exception as e:
+            logger.debug(f"Final captcha click failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"Human-like method failed: {e}")
+        return False
+
+
+async def _bypass_method_cdp_solve(page) -> bool:
+    """CDP Mode with solve_captcha() - auto-detects challenge type."""
+    try:
+        logger.debug("Attempting bypass: CDP solve_captcha")
+        await page.solve_captcha()
+        await asyncio.sleep(random.uniform(3, 5))
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP solve_captcha failed: {e}")
+        return False
+
+
+CDP_CLICK_SELECTORS = [
+    "#turnstile-widget div",      # Cloudflare Turnstile
+    "#cf-turnstile div",          # Alternative CF Turnstile
+    "iframe[src*='challenges']",  # CF challenge iframe
+    "input[type='checkbox']",     # Generic checkbox (DDOS-Guard)
+    "[class*='checkbox']",        # Class-based checkbox
+    "#challenge-running",         # CF challenge indicator
+]
+
+
+async def _bypass_method_cdp_click(page) -> bool:
+    """CDP Mode with native clicking - no PyAutoGUI dependency."""
+    try:
+        logger.debug("Attempting bypass: CDP native click")
+
+        for selector in CDP_CLICK_SELECTORS:
+            try:
+                if not await page.is_element_visible(selector):
+                    continue
+
+                logger.debug(f"CDP clicking: {selector}")
+                await page.click(selector)
+                await asyncio.sleep(random.uniform(2, 4))
+
+                if await _is_bypassed(page):
+                    return True
+            except Exception as e:
+                logger.debug(f"CDP click on '{selector}' failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP Mode click failed: {e}")
+        return False
+
+
+CDP_GUI_CLICK_SELECTORS = [
+    "#turnstile-widget div",      # Cloudflare Turnstile
+    "#cf-turnstile div",          # Alternative CF Turnstile
+    "#challenge-stage div",       # CF challenge stage
+    "input[type='checkbox']",     # Generic checkbox
+    "[class*='cb-i']",            # DDOS-Guard checkbox
+]
+
+
+async def _bypass_method_cdp_gui_click(page) -> bool:
+    """CDP Mode with gui_click-style behavior."""
+    try:
+        logger.debug("Attempting bypass: CDP gui_click (mouse-based)")
+
+        try:
+            logger.debug("Trying solve_captcha()")
+            await page.solve_captcha()
+            await asyncio.sleep(random.uniform(3, 5))
+
+            if await _is_bypassed(page):
+                return True
+        except Exception as e:
+            logger.debug(f"solve_captcha() failed: {e}")
+
+        for selector in CDP_GUI_CLICK_SELECTORS:
+            try:
+                if not await page.is_element_visible(selector):
+                    continue
+
+                logger.debug(f"CDP click_with_offset: {selector}")
+                await page.click_with_offset(selector, 0, 0, center=True)
+                await asyncio.sleep(random.uniform(3, 5))
+
+                if await _is_bypassed(page):
+                    return True
+            except Exception as e:
+                logger.debug(f"CDP gui_click on '{selector}' failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP Mode gui_click failed: {e}")
+        return False
+
+
+BYPASS_METHODS = [
+    _bypass_method_cdp_solve,
+    _bypass_method_cdp_gui_click,
+    _bypass_method_cdp_click,
+    _bypass_method_humanlike,
+]
+
+MAX_CONSECUTIVE_SAME_CHALLENGE = 3
+
+
+def _check_cancellation(cancel_flag: Optional[Event], message: str) -> None:
+    """Check if cancellation was requested and raise if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(message)
+        raise BypassCancelledException("Bypass cancelled")
+
+
+async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional[Event] = None) -> bool:
+    """Attempt to bypass Cloudflare/DDOS-Guard protection using multiple methods."""
+    max_retries = max_retries if max_retries is not None else app_config.MAX_RETRY
+
+    last_challenge_type = None
+    consecutive_same_challenge = 0
+    # Allow at least one full pass through all bypass methods before aborting due to a "stuck" challenge.
+    min_same_challenge_before_abort = max(MAX_CONSECUTIVE_SAME_CHALLENGE, len(BYPASS_METHODS) + 1)
+
+    for try_count in range(max_retries):
+        _check_cancellation(cancel_flag, "Bypass cancelled by user")
+
+        if await _is_bypassed(page):
+            if try_count == 0:
+                logger.info("Page already bypassed")
+            return True
+
+        challenge_type = await _detect_challenge_type(page)
+        logger.debug(f"Challenge detected: {challenge_type}")
+
+        # No challenge detected but page doesn't look bypassed - wait and retry
+        if challenge_type == "none":
+            logger.info("No challenge detected, waiting for page to settle...")
+            await asyncio.sleep(random.uniform(2, 3))
+            if await _is_bypassed(page):
+                return True
+            # Try a simple refresh instead of captcha methods
+            try:
+                await page.reload(ignore_cache=True)
+                await asyncio.sleep(random.uniform(1, 2))
+                if await _is_bypassed(page):
+                    logger.info("Bypass successful after refresh")
+                    return True
+            except Exception as e:
+                logger.debug(f"Refresh during no-challenge wait failed: {e}")
+            continue
+
+        if challenge_type == last_challenge_type:
+            consecutive_same_challenge += 1
+            if consecutive_same_challenge >= min_same_challenge_before_abort:
+                logger.warning(
+                    f"Same challenge ({challenge_type}) detected {consecutive_same_challenge} times - aborting"
+                )
+                return False
+        else:
+            consecutive_same_challenge = 1
+        last_challenge_type = challenge_type
+
+        method = BYPASS_METHODS[try_count % len(BYPASS_METHODS)]
+        logger.info(f"Bypass attempt {try_count + 1}/{max_retries} using {method.__name__}")
+
+        if try_count > 0:
+            wait_time = min(random.uniform(2, 4) * try_count, 12)
+            logger.info(f"Waiting {wait_time:.1f}s before trying...")
+            for _ in range(int(wait_time)):
+                _check_cancellation(cancel_flag, "Bypass cancelled during wait")
+                await asyncio.sleep(1)
+            await asyncio.sleep(wait_time - int(wait_time))
+
+        try:
+            if await method(page):
+                logger.info(f"Bypass successful using {method.__name__}")
+                return True
+        except BypassCancelledException:
+            raise
+        except Exception as e:
+            logger.warning(f"Exception in {method.__name__}: {e}")
+
+        logger.info(f"Bypass method {method.__name__} failed.")
+
+    logger.warning("Exceeded maximum retries. Bypass failed.")
+    return False
+
+def _get_browser_args() -> list[str]:
+    """Build extra Chrome arguments, pre-resolving hostnames via patched DNS.
+
+    Pre-resolves AA hostnames and passes IPs to Chrome via --host-resolver-rules,
+    bypassing Chrome's DNS entirely for those hosts.
+    """
+    arguments = [
+        "--ignore-certificate-errors",
+        "--ignore-ssl-errors",
+        "--allow-running-insecure-content",
+        "--ignore-certificate-errors-spki-list",
+        "--ignore-certificate-errors-skip-list",
+        # Chrome 144+ disabled automatic SwiftShader fallback for WebGL (security reasons).
+        # Without this flag, WebGL is broken in headless/Docker which triggers bot detection.
+        # See: https://issues.chromium.org/issues/40277080
+        "--enable-unsafe-swiftshader",
+    ]
+
+    if app_config.get("DEBUG", False):
+        arguments.extend([
+            "--enable-logging",
+            "--v=1",
+            "--log-file=" + str(LOG_DIR / "chrome_browser.log")
+        ])
+
+    host_rules = _build_host_resolver_rules()
+    if host_rules:
+        arguments.append(f'--host-resolver-rules={", ".join(host_rules)}')
+        logger.debug(f"Chrome: Using host resolver rules for {len(host_rules)} hosts")
+    else:
+        logger.warning("Chrome: No hosts could be pre-resolved")
+
+    return arguments
+
+
+def _build_host_resolver_rules() -> list[str]:
+    """Pre-resolve AA hostnames and build Chrome host resolver rules."""
+    host_rules = []
+
+    try:
+        for url in network.get_available_aa_urls():
+            hostname = urlparse(url).hostname
+            if not hostname:
+                continue
+
+            try:
+                results = socket.getaddrinfo(hostname, 443, socket.AF_INET)
+                if results:
+                    ip = results[0][4][0]
+                    host_rules.append(f"MAP {hostname} {ip}")
+                    logger.debug(f"Chrome: Pre-resolved {hostname} -> {ip}")
+                else:
+                    logger.warning(f"Chrome: No addresses returned for {hostname}")
+            except socket.gaierror as e:
+                logger.warning(f"Chrome: Could not pre-resolve {hostname}: {e}")
+    except Exception as e:
+        logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}")
+
+    return host_rules
+
+DRIVER_RESET_ERRORS = {"ProtocolException", "RuntimeError", "TimeoutError"}
+
+
+async def _get(url: str, driver, cancel_flag: Optional[Event] = None) -> str:
+    """Fetch URL with Cloudflare bypass using a CDP browser."""
+    _check_cancellation(cancel_flag, "Bypass cancelled before starting")
+
+    logger.debug(f"CDP_GET: {url}")
+
+    logger.debug("Opening URL with SeleniumBase CDP...")
+    page = await driver.get(url)
+    try:
+        await page.wait()
+    except Exception:
+        pass
+
+    _check_cancellation(cancel_flag, "Bypass cancelled after page load")
+
+    try:
+        current_url = await page.get_current_url()
+        title = await page.get_title()
+        logger.debug(f"Page loaded - URL: {current_url}, Title: {title}")
+    except Exception as e:
+        logger.debug(f"Could not get page info: {e}")
+
+    logger.debug("Starting bypass process...")
+    if await _bypass(page, cancel_flag=cancel_flag):
+        await _extract_cookies_from_cdp(driver, page, url)
+        return await page.get_page_source()
+
+    logger.warning("Bypass completed but page still shows protection")
+    try:
+        body = await page.evaluate("document.body ? document.body.innerText : ''")
+        if body:
+            logger.debug(f"Page content: {body[:500]}..." if len(body) > 500 else body)
+    except Exception:
+        pass
+
+    return ""
+
+
+def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = None) -> str:
+    """Fetch a URL with protection bypass. Creates fresh Chrome instance for each bypass."""
+    retry = retry if retry is not None else app_config.MAX_RETRY
+
+    with LOCKED:
+        # Try cookies first - another request may have completed bypass while waiting
+        cached_result = _try_with_cached_cookies(url, urlparse(url).hostname or "")
+        if cached_result:
+            return cached_result
+
+        async def _run_bypass() -> str:
+            driver = None
+            try:
+                driver = await _create_cdp_browser(url)
+
+                for attempt in range(retry):
+                    _check_cancellation(cancel_flag, "Bypass cancelled before attempt")
+
+                    try:
+                        result = await _get(url, driver, cancel_flag)
+                        if result:
+                            return result
+                    except BypassCancelledException:
+                        raise
+                    except Exception as e:
+                        error_details = f"{type(e).__name__}: {e}"
+                        logger.warning(f"Bypass failed (attempt {attempt + 1}/{retry}): {error_details}")
+                        logger.debug(f"Stack trace: {traceback.format_exc()}")
+
+                        # On CDP errors, quit and create a fresh browser
+                        if type(e).__name__ in DRIVER_RESET_ERRORS:
+                            logger.info("Restarting Chrome due to browser error...")
+                            await _close_cdp_driver(driver)
+                            driver = await _create_cdp_browser(url)
+
+                logger.error(f"Bypass failed after {retry} attempts")
+                return ""
+            finally:
+                if driver:
+                    await _close_cdp_driver(driver)
+
+        return _CDP_WORKER.run(_run_bypass())
+
+def _get_proxy_string(url: str) -> Optional[str]:
+    """Return a single proxy string for CDP, honoring NO_PROXY."""
+    proxies = get_proxies(url)
+    if not proxies:
+        return None
+    proxy_url = proxies.get("https") or proxies.get("http")
+    return proxy_url or None
+
+
+async def _create_cdp_browser(url: str) -> Any:
+    """Create a fresh CDP browser instance."""
+    browser_args = _get_browser_args()
+    screen_width, screen_height = get_screen_size()
+    display_width = screen_width + 100
+    display_height = screen_height + 150
+    proxy = _get_proxy_string(url)
+
+    logger.debug(f"Creating Pure CDP browser with args: {browser_args}")
+    logger.debug(f"Browser screen size: {screen_width}x{screen_height}")
+
+    driver = await cdp_driver.start_async(
+        headless=False,
+        headed=False,
+        xvfb=True,
+        xvfb_metrics=f"{display_width},{display_height}",
+        sandbox=False,
+        lang="en",
+        incognito=True,
+        ad_block=True,
+        proxy=proxy,
+        browser_args=browser_args,
+    )
+
+    try:
+        await driver.page.set_window_rect(0, 0, screen_width, screen_height)
+    except Exception as e:
+        logger.debug(f"Failed to set window size: {e}")
+
+    # Start FFmpeg recording if debug mode (record each bypass session)
+    if app_config.get("DEBUG", False) and not DISPLAY.get("ffmpeg"):
+        _start_ffmpeg_recording(display=os.environ.get("DISPLAY", ":0"))
+
+    await asyncio.sleep(app_config.DEFAULT_SLEEP)
+    logger.info("Chrome browser ready (Pure CDP)")
+    logger.log_resource_usage()
+    return driver
+
+
+async def _close_cdp_driver(driver) -> None:
+    """Close CDP connections and stop the browser."""
+    if not driver:
+        return
+
+    logger.debug("Quitting Chrome browser (CDP)...")
+
+    _stop_ffmpeg_recording()
+
+    try:
+        connections = []
+        if hasattr(driver, "connection") and driver.connection:
+            connections.append(driver.connection)
+        if hasattr(driver, "targets") and driver.targets:
+            connections.extend(driver.targets)
+        for conn in connections:
+            try:
+                await conn.aclose()
+            except Exception as e:
+                logger.debug(f"Failed to close websocket connection: {e}")
+    except Exception as e:
+        logger.debug(f"Error during connection cleanup: {e}")
+
+    try:
+        driver.stop()
+        logger.debug("Stopped CDP browser")
+    except Exception as e:
+        logger.debug(f"CDP stop: {e}")
+
+    if env.DOCKERMODE:
+        await asyncio.sleep(0.3)
+        try:
+            pid = getattr(driver, "_process_pid", None)
+
+            def _pid_alive(check_pid: int) -> bool:
+                try:
+                    os.kill(check_pid, 0)
+                except ProcessLookupError:
+                    return False
+                except PermissionError:
+                    return True
+                return True
+
+            if pid and _pid_alive(pid):
+                try:
+                    os.kill(pid, signal.SIGTERM)
+                    await asyncio.sleep(0.1)
+                    if _pid_alive(pid):
+                        os.kill(pid, signal.SIGKILL)
+                    logger.debug(f"Killed Chrome pid {pid}")
+                except Exception as e:
+                    logger.debug(f"Failed to kill Chrome pid {pid}: {e}")
+        except Exception as e:
+            logger.debug(f"Process cleanup failed: {e}")
+
+    logger.log_resource_usage()
+
+
+def _start_ffmpeg_recording(display: str) -> None:
+    """Start FFmpeg screen recording for debug mode."""
+    global DISPLAY
+    RECORDING_DIR.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
+    output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
+
+    screen_width, screen_height = get_screen_size()
+    display_width = screen_width + 100
+    display_height = screen_height + 150
+
+    ffmpeg_cmd = [
+        "ffmpeg", "-y", "-f", "x11grab",
+        "-video_size", f"{display_width}x{display_height}",
+        "-i", display,
+        "-c:v", "libx264", "-preset", "ultrafast",
+        "-maxrate", "700k", "-bufsize", "1400k", "-crf", "36",
+        "-pix_fmt", "yuv420p", "-tune", "animation",
+        "-x264-params", "bframes=0:deblock=-1,-1",
+        "-r", "15", "-an",
+        output_file.as_posix(),
+        "-nostats", "-loglevel", "0"
+    ]
+    logger.debug("Starting FFmpeg recording to %s", output_file)
+    logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
+    DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
+    DISPLAY["ffmpeg_output"] = output_file
+
+
+def _stop_ffmpeg_recording() -> None:
+    """Stop FFmpeg screen recording if running."""
+    import signal
+    global DISPLAY
+    proc = DISPLAY.get("ffmpeg")
+    output_file = DISPLAY.get("ffmpeg_output")
+    if not proc:
+        return
+    if proc.poll() is not None:
+        logger.debug("FFmpeg already stopped")
+        DISPLAY["ffmpeg"] = None
+        DISPLAY["ffmpeg_output"] = None
+        return
+    try:
+        proc.send_signal(signal.SIGINT)
+        proc.wait(timeout=5)
+        logger.debug("Stopped ffmpeg recording")
+    except Exception as e:
+        logger.debug(f"ffmpeg stop: {e}")
+        try:
+            proc.terminate()
+            proc.wait(timeout=2)
+        except Exception:
+            pass
+        try:
+            proc.kill()
+        except Exception:
+            pass
+    DISPLAY["ffmpeg"] = None
+    DISPLAY["ffmpeg_output"] = None
+
+
+def _try_with_cached_cookies(url: str, hostname: str) -> Optional[str]:
+    """Attempt request with cached cookies before using Chrome."""
+    cookies = get_cf_cookies_for_domain(hostname)
+    if not cookies:
+        return None
+
+    try:
+        headers = {}
+        stored_ua = get_cf_user_agent_for_domain(hostname)
+        if stored_ua:
+            headers['User-Agent'] = stored_ua
+
+        logger.debug(f"Trying request with cached cookies: {url}")
+        response = requests.get(url, cookies=cookies, headers=headers, proxies=get_proxies(url), timeout=(5, 10), verify=get_ssl_verify(url))
+        if response.status_code == 200:
+            logger.debug("Cached cookies worked, skipped Chrome bypass")
+            return response.text
+    except Exception:
+        pass
+
+    return None
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional[network.AAMirrorSelector] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML content from a URL using the internal Cloudflare Bypasser."""
+    sel = selector or network.AAMirrorSelector()
+    attempt_url = sel.rewrite(url)
+    hostname = urlparse(attempt_url).hostname or ""
+
+    cached_result = _try_with_cached_cookies(attempt_url, hostname)
+    if cached_result:
+        return cached_result
+
+    try:
+        response_html = get(attempt_url, cancel_flag=cancel_flag)
+    except BypassCancelledException:
+        raise
+    except Exception:
+        _check_cancellation(cancel_flag, "Bypass cancelled")
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            attempt_url = sel.rewrite(url)
+            response_html = get(attempt_url, cancel_flag=cancel_flag)
+        else:
+            raise
+
+    if not response_html.strip():
+        raise requests.exceptions.RequestException("Failed to bypass Cloudflare")
+
+    return response_html

shelfmark/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration module - environment variables and settings."""

shelfmark/config/booklore_settings.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.outputs.booklore import (
+    BookloreConfig,
+    BookloreError,
+    booklore_list_libraries,
+    booklore_login,
+)
+
+logger = setup_logger(__name__)
+
+_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
+    "key": None,
+    "library_options": [],
+    "path_options": [],
+}
+
+
+def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
+    return f"{base_url}|{username}|{hash(password)}"
+
+
+def _get_booklore_select_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    # library_id/path_id are not used for login/library listing
+    booklore_config = BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=1,
+        path_id=1,
+        verify_tls=True,
+        refresh_after_upload=True,
+    )
+
+    token = booklore_login(booklore_config)
+    libraries = booklore_list_libraries(booklore_config, token) or []
+    logger.debug("Booklore libraries response: %s", libraries)
+
+    library_options: list[dict[str, Any]] = []
+    path_options: list[dict[str, Any]] = []
+
+    for library in libraries:
+        if not isinstance(library, dict):
+            continue
+
+        library_id = library.get("id")
+        if library_id is None:
+            continue
+
+        library_name = str(library.get("name") or f"Library {library_id}")
+        library_id_str = str(library_id)
+
+        library_options.append({"value": library_id_str, "label": library_name})
+
+        paths = library.get("paths") or []
+        if not isinstance(paths, list):
+            continue
+
+        for path in paths:
+            if not isinstance(path, dict):
+                continue
+
+            path_id = path.get("id")
+            if path_id is None:
+                continue
+
+            path_label = str(path.get("path") or f"Path {path_id}")
+            path_options.append(
+                {
+                    "value": str(path_id),
+                    "label": f"{library_name}: {path_label}",
+                    "childOf": library_id_str,
+                }
+            )
+
+    logger.debug(
+        "Booklore options built: libraries=%d paths=%d",
+        len(library_options),
+        len(path_options),
+    )
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    _BOOKLORE_OPTIONS_CACHE.update(
+        {
+            "key": cache_key,
+            "library_options": library_options,
+            "path_options": path_options,
+        }
+    )
+
+    return library_options, path_options
+
+
+def _get_booklore_cached_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+        return (
+            _BOOKLORE_OPTIONS_CACHE.get("library_options", []),
+            _BOOKLORE_OPTIONS_CACHE.get("path_options", []),
+        )
+
+    return _get_booklore_select_options(base_url, username, password)
+
+
+def get_booklore_library_options() -> list[dict[str, Any]]:
+    """Build Booklore library options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        library_options, _ = _get_booklore_cached_options(base_url, username, password)
+        return library_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore libraries: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
+        return []
+
+
+def get_booklore_path_options() -> list[dict[str, Any]]:
+    """Build Booklore path options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        _, path_options = _get_booklore_cached_options(base_url, username, password)
+        return path_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore paths: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
+        return []
+
+
+def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test the Booklore connection using current form values."""
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
+    password = _get_value("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        return {"success": False, "message": "Booklore URL is required"}
+    if not username:
+        return {"success": False, "message": "Booklore username is required"}
+    if not password:
+        return {"success": False, "message": "Booklore password is required"}
+
+    try:
+        library_options, _ = _get_booklore_select_options(base_url, username, password)
+
+        message = "Connected to Booklore"
+        if library_options:
+            message = f"Connected to Booklore ({len(library_options)} libraries)"
+
+        return {"success": True, "message": message}
+    except BookloreError as exc:
+        return {"success": False, "message": str(exc)}

shelfmark/config/email_settings.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.download.outputs.email import EmailOutputError, build_email_smtp_config, test_smtp_connection
+
+
+def test_email_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test SMTP connectivity using current form values (including unsaved changes)."""
+
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    settings = {
+        "EMAIL_SMTP_HOST": _get_value("EMAIL_SMTP_HOST", ""),
+        "EMAIL_SMTP_PORT": _get_value("EMAIL_SMTP_PORT", 587),
+        "EMAIL_SMTP_SECURITY": _get_value("EMAIL_SMTP_SECURITY", "starttls"),
+        "EMAIL_SMTP_USERNAME": _get_value("EMAIL_SMTP_USERNAME", ""),
+        "EMAIL_SMTP_PASSWORD": _get_value("EMAIL_SMTP_PASSWORD", ""),
+        "EMAIL_FROM": _get_value("EMAIL_FROM", ""),
+        "EMAIL_SUBJECT_TEMPLATE": _get_value("EMAIL_SUBJECT_TEMPLATE", "{Title}"),
+        "EMAIL_SMTP_TIMEOUT_SECONDS": _get_value("EMAIL_SMTP_TIMEOUT_SECONDS", 60),
+        "EMAIL_ALLOW_UNVERIFIED_TLS": _get_value("EMAIL_ALLOW_UNVERIFIED_TLS", False),
+    }
+
+    try:
+        smtp_config = build_email_smtp_config(settings)
+        test_smtp_connection(smtp_config)
+        return {"success": True, "message": "Connected to SMTP server"}
+    except EmailOutputError as exc:
+        return {"success": False, "message": str(exc)}
+    except Exception as exc:
+        return {"success": False, "message": f"SMTP test failed: {exc}"}
+

shelfmark/config/env.py

@@ -0,0 +1,164 @@
+"""Bootstrap environment variables. No local dependencies - import first."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+
+def string_to_bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ["true", "yes", "1", "y"]
+
+
+def _read_debug_from_config() -> bool:
+    """Read DEBUG from env var or config file (import-time safe)."""
+    env_debug = os.environ.get("DEBUG")
+    if env_debug is not None:
+        return string_to_bool(env_debug)
+
+    # Try to read from config file
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    config_file = config_dir / "plugins" / "advanced.json"
+
+    if config_file.exists():
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+                if "DEBUG" in config:
+                    return bool(config["DEBUG"])
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return False
+
+
+def _is_sqlite_file(path: Path) -> bool:
+    """Check if a file is a valid SQLite database by reading magic bytes."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(16)
+            return header[:16] == b"SQLite format 3\x00"
+    except (OSError, PermissionError):
+        return False
+
+
+def _resolve_cwa_db_path() -> Path | None:
+    """Resolve CWA database path from env var or default location."""
+    env_path = os.getenv("CWA_DB_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file() and _is_sqlite_file(path):
+            return path
+
+    # Check default mount path
+    default_path = Path("/auth/app.db")
+    if default_path.exists() and default_path.is_file() and _is_sqlite_file(default_path):
+        return default_path
+
+    return None
+
+
+def _is_config_dir_writable() -> bool:
+    """Check if the config directory exists and is writable."""
+    try:
+        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
+            return False
+        test_file = CONFIG_DIR / ".write_test"
+        test_file.touch()
+        test_file.unlink()
+        return True
+    except (OSError, PermissionError):
+        return False
+
+
+def is_covers_cache_enabled() -> bool:
+    """Check if cover caching is enabled (requires setting + writable config dir)."""
+    from shelfmark.core.config import config
+    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
+    return setting_enabled and _is_config_dir_writable()
+
+
+# =============================================================================
+# Bootstrap paths - needed before settings registry is available
+# =============================================================================
+
+CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
+LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
+LOG_DIR = LOG_ROOT / "shelfmark"
+LOG_FILE = LOG_DIR / "shelfmark.log"
+TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
+INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
+
+
+# =============================================================================
+# Logger configuration - needed before settings registry is available
+# =============================================================================
+
+DEBUG = _read_debug_from_config()
+LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
+ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+
+
+# =============================================================================
+# Flask configuration - needed before app starts
+# =============================================================================
+
+FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
+
+
+# =============================================================================
+# Authentication
+# =============================================================================
+
+SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
+SESSION_COOKIE_NAME = "shelfmark_session"
+CWA_DB_PATH = _resolve_cwa_db_path()
+HIDE_LOCAL_AUTH = string_to_bool(os.getenv("HIDE_LOCAL_AUTH", "false"))
+OIDC_AUTO_REDIRECT = string_to_bool(os.getenv("OIDC_AUTO_REDIRECT", "false"))
+
+
+# =============================================================================
+# Version information from Docker build
+# =============================================================================
+
+BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
+RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
+
+
+# =============================================================================
+# Capability detection - runtime checks, not user-configurable
+# =============================================================================
+
+DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
+TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
+USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
+
+
+# =============================================================================
+# Onboarding
+# =============================================================================
+
+# Set to false to skip the onboarding wizard entirely (useful for ephemeral storage)
+ONBOARDING = string_to_bool(os.getenv("ONBOARDING", "true"))
+
+
+# =============================================================================
+# Debug/development settings
+# =============================================================================
+
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
+
+
+# =============================================================================
+# Legacy migration support - will be removed in future version
+# =============================================================================
+
+# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
+# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
+_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))

shelfmark/config/migrations.py

@@ -0,0 +1,135 @@
+"""Configuration migration helpers."""
+
+import json
+from typing import Any, Callable
+
+
+_DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
+    "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
+    "CWA_RESTRICT_SETTINGS_TO_ADMIN",
+    "RESTRICT_SETTINGS_TO_ADMIN",
+)
+
+
+def _as_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+    return bool(value)
+
+
+def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None:
+    """Pick the best legacy admin-restriction value to migrate."""
+    auth_method = str(config.get("AUTH_METHOD", "")).strip().lower()
+
+    if (
+        auth_method == "proxy"
+        and "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config
+    ):
+        return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if auth_method == "cwa" and "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    return None
+
+
+def migrate_security_settings(
+    *,
+    load_security_config: Callable[[], dict[str, Any]],
+    load_users_config: Callable[[], dict[str, Any]],
+    save_users_config: Callable[[dict[str, Any]], None],
+    ensure_config_dir: Callable[[], None],
+    get_config_path: Callable[[], Any],
+    sync_builtin_admin_user: Callable[[str, str], None],
+    logger: Any,
+) -> None:
+    """Migrate legacy security keys and sync builtin admin credentials."""
+    try:
+        config = load_security_config()
+        users_config = load_users_config()
+        migrated_security = False
+        migrated_users = False
+
+        if "USE_CWA_AUTH" in config:
+            old_value = config.pop("USE_CWA_AUTH")
+            if "AUTH_METHOD" not in config:
+                if old_value:
+                    config["AUTH_METHOD"] = "cwa"
+                    logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
+                else:
+                    if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                        config["AUTH_METHOD"] = "builtin"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
+                    else:
+                        config["AUTH_METHOD"] = "none"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
+                migrated_security = True
+            else:
+                logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
+                migrated_security = True
+
+        # Backfill AUTH_METHOD for configs that have builtin credentials but
+        # were never migrated from USE_CWA_AUTH (e.g. dev builds that predated
+        # the AUTH_METHOD field).
+        if "AUTH_METHOD" not in config:
+            if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                config["AUTH_METHOD"] = "builtin"
+                migrated_security = True
+                logger.info(
+                    "Backfilled AUTH_METHOD='builtin' from legacy "
+                    "BUILTIN_USERNAME/BUILTIN_PASSWORD_HASH credentials"
+                )
+
+        if "RESTRICT_SETTINGS_TO_ADMIN" not in users_config:
+            legacy_restrict = _pick_legacy_settings_restriction(config)
+            if legacy_restrict is not None:
+                save_users_config({"RESTRICT_SETTINGS_TO_ADMIN": legacy_restrict})
+                migrated_users = True
+                logger.info(
+                    "Migrated legacy settings-admin restriction to users.RESTRICT_SETTINGS_TO_ADMIN="
+                    f"{legacy_restrict}"
+                )
+
+        for deprecated_key in _DEPRECATED_SETTINGS_RESTRICTION_KEYS:
+            if deprecated_key in config:
+                config.pop(deprecated_key, None)
+                migrated_security = True
+                logger.info(f"Removed deprecated security setting: {deprecated_key}")
+
+        try:
+            sync_builtin_admin_user(
+                config.get("BUILTIN_USERNAME", ""),
+                config.get("BUILTIN_PASSWORD_HASH", ""),
+            )
+        except Exception as exc:
+            logger.error(
+                "Failed to sync builtin credentials to users database during migration: "
+                f"{exc}"
+            )
+
+        if migrated_security:
+            ensure_config_dir()
+            config_path = get_config_path()
+            with open(config_path, "w") as f:
+                json.dump(config, f, indent=2)
+            logger.info("Security settings migration completed successfully")
+        elif migrated_users:
+            logger.info("Users settings migration completed successfully")
+        else:
+            logger.debug("No security settings migration needed")
+
+    except FileNotFoundError:
+        logger.debug("No existing security config file found - nothing to migrate")
+    except Exception as exc:
+        logger.error(f"Failed to migrate security settings: {exc}")

shelfmark/config/notifications_settings.py

@@ -0,0 +1,337 @@
+"""Notifications settings tab registration."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlsplit
+
+from shelfmark.core.notifications import NotificationEvent, send_test_notification
+from shelfmark.core.settings_registry import (
+    ActionButton,
+    HeadingField,
+    TableField,
+    load_config_file,
+    register_on_save,
+    register_settings,
+)
+
+_URL_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*$")
+
+_ROUTE_EVENT_ALL = "all"
+_ADMIN_EVENT_OPTIONS = [
+    {"value": NotificationEvent.REQUEST_CREATED.value, "label": "New request submitted"},
+    {"value": NotificationEvent.REQUEST_FULFILLED.value, "label": "Request approved"},
+    {"value": NotificationEvent.REQUEST_REJECTED.value, "label": "Request rejected"},
+    {"value": NotificationEvent.DOWNLOAD_COMPLETE.value, "label": "Download complete"},
+    {"value": NotificationEvent.DOWNLOAD_FAILED.value, "label": "Download failed"},
+]
+_ROUTE_EVENT_OPTIONS = [
+    {"value": _ROUTE_EVENT_ALL, "label": "All"},
+    *_ADMIN_EVENT_OPTIONS,
+]
+_ROUTE_EVENT_ORDER = [option["value"] for option in _ROUTE_EVENT_OPTIONS]
+_ROUTE_EVENT_INDEX = {event: index for index, event in enumerate(_ROUTE_EVENT_ORDER)}
+_ALLOWED_ROUTE_EVENTS = set(_ROUTE_EVENT_ORDER)
+
+_DEFAULT_ROUTE_ROWS = [{"event": [_ROUTE_EVENT_ALL], "url": ""}]
+
+
+def _looks_like_apprise_url(url: str) -> bool:
+    split = urlsplit(url)
+    if not split.scheme:
+        return False
+    if not _URL_SCHEME_RE.match(split.scheme):
+        return False
+    return " " not in url
+
+
+def _coerce_route_rows(value: Any) -> list[dict[str, Any]]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [row for row in value if isinstance(row, dict)]
+    if isinstance(value, dict):
+        return [value]
+    return []
+
+
+def _coerce_route_event_values(value: Any) -> list[Any]:
+    if isinstance(value, list):
+        return value
+    if isinstance(value, (tuple, set)):
+        return list(value)
+    return [value]
+
+
+def _normalize_route_events(value: Any) -> list[str]:
+    normalized: list[str] = []
+    seen: set[str] = set()
+
+    for raw_event in _coerce_route_event_values(value):
+        event = str(raw_event or "").strip().lower()
+        if not event or event not in _ALLOWED_ROUTE_EVENTS:
+            continue
+        if event in seen:
+            continue
+        seen.add(event)
+        normalized.append(event)
+
+    if _ROUTE_EVENT_ALL in seen:
+        return [_ROUTE_EVENT_ALL]
+
+    return sorted(normalized, key=lambda event: _ROUTE_EVENT_INDEX[event])
+
+
+def _normalize_routes(value: Any) -> list[dict[str, Any]]:
+    normalized: list[dict[str, Any]] = []
+    seen: set[tuple[tuple[str, ...], str]] = set()
+
+    for row in _coerce_route_rows(value):
+        events = _normalize_route_events(row.get("event"))
+        if not events:
+            continue
+
+        url = str(row.get("url") or "").strip()
+        key = (tuple(events), url)
+        if key in seen:
+            continue
+        seen.add(key)
+
+        normalized.append({"event": events, "url": url})
+
+    return normalized
+
+
+def _count_invalid_route_events(value: Any) -> int:
+    invalid = 0
+    for row in _coerce_route_rows(value):
+        raw_events = _coerce_route_event_values(row.get("event"))
+        if not raw_events:
+            invalid += 1
+            continue
+
+        for raw_event in raw_events:
+            event = str(raw_event or "").strip().lower()
+            if not event or event not in _ALLOWED_ROUTE_EVENTS:
+                invalid += 1
+    return invalid
+
+
+def _count_invalid_route_urls(routes: list[dict[str, Any]]) -> int:
+    return sum(1 for row in routes if row["url"] and not _looks_like_apprise_url(row["url"]))
+
+
+def _ensure_default_route_row(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return routes if routes else [dict(row) for row in _DEFAULT_ROUTE_ROWS]
+
+
+def _extract_unique_route_urls(routes: list[dict[str, Any]]) -> list[str]:
+    urls: list[str] = []
+    seen: set[str] = set()
+    for row in routes:
+        url = row.get("url", "")
+        if not url:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        urls.append(url)
+    return urls
+
+
+def build_notification_test_result(routes_input: Any, *, scope_label: str) -> dict[str, Any]:
+    invalid_event_count = _count_invalid_route_events(routes_input)
+    if invalid_event_count:
+        return {
+            "success": False,
+            "message": (
+                f"Found {invalid_event_count} invalid {scope_label} notification route event value(s). "
+                "Fix route events before running a test."
+            ),
+        }
+
+    normalized_routes = _normalize_routes(routes_input)
+    invalid_url_count = _count_invalid_route_urls(normalized_routes)
+    if invalid_url_count:
+        return {
+            "success": False,
+            "message": (
+                f"Found {invalid_url_count} invalid {scope_label} notification URL(s). "
+                "Fix route URLs before running a test."
+            ),
+        }
+
+    urls = _extract_unique_route_urls(normalized_routes)
+    if not urls:
+        return {
+            "success": False,
+            "message": f"Add at least one {scope_label} notification URL route first.",
+        }
+
+    return send_test_notification(urls)
+
+
+def normalize_notification_routes(value: Any) -> list[dict[str, Any]]:
+    """Normalize route table rows for notification preferences."""
+    return _normalize_routes(value)
+
+
+def is_valid_notification_url(url: str) -> bool:
+    """Shared URL validation for notifications preferences."""
+    return _looks_like_apprise_url(url)
+
+
+def _on_save_notifications(values: dict[str, Any]) -> dict[str, Any]:
+    existing = load_config_file("notifications")
+    effective: dict[str, Any] = dict(existing)
+    effective.update(values)
+
+    admin_routes_input = effective.get("ADMIN_NOTIFICATION_ROUTES", [])
+    invalid_admin_event_count = _count_invalid_route_events(admin_routes_input)
+    if invalid_admin_event_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_admin_event_count} invalid global notification route event value(s)."
+            ),
+            "values": values,
+        }
+
+    normalized_admin_routes = _normalize_routes(admin_routes_input)
+    invalid_admin_url_count = _count_invalid_route_urls(normalized_admin_routes)
+    if invalid_admin_url_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_admin_url_count} invalid global notification URL(s). "
+                "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+            ),
+            "values": values,
+        }
+
+    user_routes_input = effective.get("USER_NOTIFICATION_ROUTES", [])
+    invalid_user_event_count = _count_invalid_route_events(user_routes_input)
+    if invalid_user_event_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_user_event_count} invalid personal notification route event value(s)."
+            ),
+            "values": values,
+        }
+
+    normalized_user_routes = _normalize_routes(user_routes_input)
+    invalid_user_url_count = _count_invalid_route_urls(normalized_user_routes)
+    if invalid_user_url_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_user_url_count} invalid personal notification URL(s). "
+                "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+            ),
+            "values": values,
+        }
+
+    admin_routes_touched = "ADMIN_NOTIFICATION_ROUTES" in values
+    if admin_routes_touched:
+        values["ADMIN_NOTIFICATION_ROUTES"] = _ensure_default_route_row(normalized_admin_routes)
+
+    user_routes_touched = "USER_NOTIFICATION_ROUTES" in values
+    if user_routes_touched:
+        values["USER_NOTIFICATION_ROUTES"] = _ensure_default_route_row(normalized_user_routes)
+
+    return {"error": False, "values": values}
+
+
+def _test_admin_notification_action(current_values: dict[str, Any]) -> dict[str, Any]:
+    persisted = load_config_file("notifications")
+    effective: dict[str, Any] = dict(persisted)
+    if isinstance(current_values, dict):
+        effective.update(current_values)
+
+    routes_input = effective.get("ADMIN_NOTIFICATION_ROUTES", [])
+    return build_notification_test_result(routes_input, scope_label="global")
+
+
+register_on_save("notifications", _on_save_notifications)
+
+
+@register_settings("notifications", "Notifications", icon="bell", order=7)
+def notifications_settings():
+    """Global notifications settings."""
+    return [
+        HeadingField(
+            key="notifications_heading",
+            title="Global Notifications",
+            description=(
+                "Global notifications send selected events for all users to configured routes. "
+                "Users can manage personal notifications in User Preferences."
+            ),
+        ),
+        TableField(
+            key="ADMIN_NOTIFICATION_ROUTES",
+            label="",
+            description=(
+                "Create one route per URL. Start with All, then add event-specific routes "
+                "for targeted delivery. Need format examples? "
+                "[View Apprise URL formats](https://appriseit.com/services/)."
+            ),
+            columns=[
+                {
+                    "key": "event",
+                    "label": "Event",
+                    "type": "multiselect",
+                    "options": _ROUTE_EVENT_OPTIONS,
+                    "defaultValue": [_ROUTE_EVENT_ALL],
+                    "placeholder": "Select events...",
+                },
+                {
+                    "key": "url",
+                    "label": "Notification URL",
+                    "type": "text",
+                    "placeholder": "e.g. ntfys://ntfy.sh/shelfmark",
+                },
+            ],
+            default=[dict(row) for row in _DEFAULT_ROUTE_ROWS],
+            add_label="Add Route",
+            empty_message="No routes configured.",
+        ),
+        ActionButton(
+            key="test_admin_notification",
+            label="Test Notification",
+            description="Send a test notification to all configured global route URLs.",
+            style="primary",
+            callback=_test_admin_notification_action,
+        ),
+        TableField(
+            key="USER_NOTIFICATION_ROUTES",
+            label="",
+            description=(
+                "Create one route per URL. Start with All, then add event-specific routes "
+                "for targeted delivery. Need format examples? "
+                "[View Apprise URL formats](https://appriseit.com/services/)."
+            ),
+            columns=[
+                {
+                    "key": "event",
+                    "label": "Event",
+                    "type": "multiselect",
+                    "options": _ROUTE_EVENT_OPTIONS,
+                    "defaultValue": [_ROUTE_EVENT_ALL],
+                    "placeholder": "Select events...",
+                },
+                {
+                    "key": "url",
+                    "label": "Notification URL",
+                    "type": "text",
+                    "placeholder": "e.g. ntfys://ntfy.sh/username-topic",
+                },
+            ],
+            default=[dict(row) for row in _DEFAULT_ROUTE_ROWS],
+            add_label="Add Route",
+            empty_message="No routes configured.",
+            user_overridable=True,
+            hidden_in_ui=True,
+        ),
+    ]

shelfmark/config/security.py

@@ -0,0 +1,284 @@
+"""Authentication settings registration."""
+
+from typing import Any, Dict, Callable
+
+from shelfmark.config.migrations import migrate_security_settings
+from shelfmark.config.security_handlers import (
+    on_save_security,
+    test_oidc_connection,
+)
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    register_settings,
+    register_on_save,
+    load_config_file,
+    TextField,
+    SelectField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+    TagListField,
+    CustomComponentField,
+)
+from shelfmark.core.user_db import sync_builtin_admin_user
+
+logger = setup_logger(__name__)
+
+
+def _auth_condition(auth_method: str) -> dict[str, str]:
+    return {"field": "AUTH_METHOD", "value": auth_method}
+
+
+def _auth_field(factory: Callable[..., Any], auth_method: str, **kwargs: Any) -> Any:
+    return factory(show_when=_auth_condition(auth_method), **kwargs)
+
+
+def _migrate_security_settings() -> None:
+    from shelfmark.core.settings_registry import (
+        _get_config_file_path,
+        _ensure_config_dir,
+        save_config_file,
+    )
+
+    migrate_security_settings(
+        load_security_config=lambda: load_config_file("security"),
+        load_users_config=lambda: load_config_file("users"),
+        save_users_config=lambda values: save_config_file("users", values),
+        ensure_config_dir=lambda: _ensure_config_dir("security"),
+        get_config_path=lambda: _get_config_file_path("security"),
+        sync_builtin_admin_user=sync_builtin_admin_user,
+        logger=logger,
+    )
+
+
+
+def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
+    return on_save_security(values)
+
+
+def _test_oidc_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
+    return test_oidc_connection(
+        load_security_config=lambda: load_config_file("security"),
+        current_values=current_values or {},
+        logger=logger,
+    )
+
+
+@register_settings("security", "Security", icon="shield", order=5)
+def security_settings():
+    """Security and authentication settings."""
+    from shelfmark.config.env import CWA_DB_PATH
+
+    cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
+
+    auth_method_options = [
+        {"label": "No Authentication", "value": "none"},
+        {"label": "Local", "value": "builtin"},
+        {"label": "Proxy Authentication", "value": "proxy"},
+        {"label": "OIDC (OpenID Connect)", "value": "oidc"},
+        {"label": "Calibre-Web Database", "value": "cwa"},
+    ]
+
+    fields = [
+        SelectField(
+            key="AUTH_METHOD",
+            label="Authentication Method",
+            description="Select the authentication method for accessing Shelfmark.",
+            options=auth_method_options,
+            default="none",
+        ),
+        CustomComponentField(
+            key="builtin_admin_requirement",
+            component="oidc_admin_hint",
+            label=(
+                "Local authentication is inactive until a local admin account with a "
+                "password is created."
+            ),
+            show_when=_auth_condition("builtin"),
+        ),
+        CustomComponentField(
+            key="oidc_admin_requirement",
+            component="oidc_admin_hint",
+            label="A local admin account is required before OIDC can be enabled.",
+            show_when=_auth_condition("oidc"),
+        ),
+        *([] if cwa_db_available else [
+            CustomComponentField(
+                key="cwa_db_missing",
+                component="oidc_admin_hint",
+                label=(
+                    "Calibre-Web database not detected. Mount your app.db to "
+                    "/auth/app.db to enable this method. Authentication will fall "
+                    "back to none until the database is available."
+                ),
+                show_when=_auth_condition("cwa"),
+            ),
+        ]),
+        ActionButton(
+            key="open_users_tab",
+            label="Go to Users",
+            description="Configure local users and admin access in the Users tab.",
+            style="primary",
+            show_when={"field": "AUTH_METHOD", "value": ["builtin", "oidc"]},
+        ),
+        _auth_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_USER_HEADER",
+            label="Proxy Auth User Header",
+            description="The HTTP header your proxy uses to pass the authenticated username.",
+            placeholder="e.g. X-Auth-User",
+            default="X-Auth-User",
+        ),
+        _auth_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_LOGOUT_URL",
+            label="Proxy Auth Logout URL",
+            description="The URL to redirect users to for logging out. Leave empty to disable logout functionality.",
+            placeholder="https://myauth.example.com/logout",
+            default="",
+        ),
+        _auth_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_ADMIN_GROUP_HEADER",
+            label="Proxy Auth Admin Group Header",
+            description="Optional: header your proxy uses to pass user groups/roles.",
+            placeholder="e.g. X-Auth-Groups",
+            default="X-Auth-Groups",
+        ),
+        _auth_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_ADMIN_GROUP_NAME",
+            label="Proxy Auth Admin Group",
+            description="Optional: users in this group are treated as admins. Leave blank to skip group-based admin detection.",
+            placeholder="e.g. admins",
+            default="",
+        ),
+    ]
+
+    fields.append(
+        CustomComponentField(
+            key="oidc_callback_url",
+            component="settings_label",
+            label="Callback URL",
+            description="{origin}/api/auth/oidc/callback",
+            show_when=_auth_condition("oidc"),
+        )
+    )
+
+    oidc_specs = [
+        (
+            TextField,
+            {
+                "key": "OIDC_DISCOVERY_URL",
+                "label": "Discovery URL",
+                "description": "OpenID Connect discovery endpoint URL. Usually ends with /.well-known/openid-configuration.",
+                "placeholder": "https://auth.example.com/.well-known/openid-configuration",
+                "required": True,
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_CLIENT_ID",
+                "label": "Client ID",
+                "description": "OAuth2 client ID from your identity provider.",
+                "placeholder": "shelfmark",
+                "required": True,
+            },
+        ),
+        (
+            PasswordField,
+            {
+                "key": "OIDC_CLIENT_SECRET",
+                "label": "Client Secret",
+                "description": "OAuth2 client secret from your identity provider.",
+                "required": True,
+            },
+        ),
+        (
+            TagListField,
+            {
+                "key": "OIDC_SCOPES",
+                "label": "Scopes",
+                "description": "OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization.",
+                "default": ["openid", "email", "profile"],
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_GROUP_CLAIM",
+                "label": "Group Claim Name",
+                "description": "The name of the claim in the ID token that contains user groups.",
+                "placeholder": "groups",
+                "default": "groups",
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_ADMIN_GROUP",
+                "label": "Admin Group Name",
+                "description": "Users in this group will be given admin access (if enabled below). Leave empty to use database roles only.",
+                "placeholder": "shelfmark-admins",
+                "default": "",
+            },
+        ),
+        (
+            CheckboxField,
+            {
+                "key": "OIDC_USE_ADMIN_GROUP",
+                "label": "Use Admin Group for Authorization",
+                "description": "When enabled, users in the Admin Group are granted admin access. When disabled, admin access is determined solely by database roles.",
+                "default": True,
+            },
+        ),
+        (
+            CheckboxField,
+            {
+                "key": "OIDC_AUTO_PROVISION",
+                "label": "Auto-Provision Users",
+                "description": "Automatically create a user account on first OIDC login. When disabled, users must be pre-created by an admin.",
+                "default": True,
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_BUTTON_LABEL",
+                "label": "Login Button Label",
+                "description": "Custom label for the OIDC sign-in button on the login page.",
+                "placeholder": "Sign in with OIDC",
+                "default": "",
+            },
+        ),
+    ]
+    fields.extend(_auth_field(factory, "oidc", **spec) for factory, spec in oidc_specs)
+    fields.append(
+        ActionButton(
+            key="test_oidc",
+            label="Test Connection",
+            description="Fetch the OIDC discovery document and validate configuration.",
+            style="primary",
+            callback=_test_oidc_connection,
+            show_when=_auth_condition("oidc"),
+        )
+    )
+    fields.append(
+        CustomComponentField(
+            key="oidc_env_info",
+            component="oidc_env_info",
+            label="Environment-Only Options",
+            description="These options can only be set via environment variables because changing them through the UI could lock you out of the application.",
+            wrap_in_field_wrapper=True,
+            show_when=_auth_condition("oidc"),
+        )
+    )
+    return fields
+
+
+register_on_save("security", _on_save_security)

shelfmark/config/security_handlers.py

@@ -0,0 +1,75 @@
+"""Operational handlers for security settings (save/actions)."""
+
+import os
+from typing import Any, Callable
+
+from shelfmark.core.utils import normalize_http_url
+from shelfmark.core.user_db import UserDB
+from shelfmark.download.network import get_ssl_verify
+
+
+_OIDC_LOCKOUT_MESSAGE = "A local admin account with a password is required before enabling OIDC. Use the 'Go to Users' button above to create one. This ensures you can still sign in if your identity provider is unavailable."
+
+
+def _has_local_password_admin() -> bool:
+    root = os.environ.get("CONFIG_DIR", "/config")
+    user_db = UserDB(os.path.join(root, "users.db"))
+    user_db.initialize()
+    return any(user.get("password_hash") and user.get("role") == "admin" for user in user_db.list_users())
+
+
+def on_save_security(
+    values: dict[str, Any],
+) -> dict[str, Any]:
+    """Validate security values before persistence."""
+    normalized_values = values.copy()
+
+    discovery_url = normalized_values.get("OIDC_DISCOVERY_URL")
+    if discovery_url is not None:
+        normalized_values["OIDC_DISCOVERY_URL"] = normalize_http_url(
+            str(discovery_url),
+            default_scheme="https",
+        )
+
+    proxy_logout_url = normalized_values.get("PROXY_AUTH_LOGOUT_URL")
+    if proxy_logout_url is not None:
+        normalized_values["PROXY_AUTH_LOGOUT_URL"] = normalize_http_url(
+            str(proxy_logout_url),
+            default_scheme="https",
+            strip_trailing_slash=False,
+        )
+
+    if normalized_values.get("AUTH_METHOD") == "oidc" and not _has_local_password_admin():
+        return {"error": True, "message": _OIDC_LOCKOUT_MESSAGE, "values": normalized_values}
+
+    return {"error": False, "values": normalized_values}
+
+
+def test_oidc_connection(
+    *,
+    load_security_config: Callable[[], dict[str, Any]],
+    current_values: dict[str, Any] | None = None,
+    logger: Any,
+) -> dict[str, Any]:
+    """Fetch and validate the configured OIDC discovery document."""
+    import requests
+
+    try:
+        # Prefer the current (unsaved) form value over the saved config
+        discovery_url = (current_values or {}).get("OIDC_DISCOVERY_URL") or load_security_config().get("OIDC_DISCOVERY_URL", "")
+        if not discovery_url:
+            return {"success": False, "message": "Discovery URL is not configured."}
+
+        response = requests.get(discovery_url, timeout=10, verify=get_ssl_verify(discovery_url))
+        response.raise_for_status()
+        document = response.json()
+
+        required_fields = ["issuer", "authorization_endpoint", "token_endpoint"]
+        missing_fields = [field for field in required_fields if field not in document]
+        if missing_fields:
+            return {"success": False, "message": f"Discovery document missing fields: {', '.join(missing_fields)}"}
+
+        return {"success": True, "message": f"Connected to {document['issuer']}"}
+    except Exception as exc:
+        logger.error(f"OIDC connection test failed: {exc}")
+        return {"success": False, "message": f"Connection failed: {str(exc)}"}

shelfmark/config/users_settings.py

@@ -0,0 +1,390 @@
+"""Users settings tab registration.
+
+This registers a 'users' tab in the settings sidebar.
+The actual user management is handled by a custom frontend component
+that talks to /api/admin/users endpoints.
+"""
+
+from typing import Any
+
+from shelfmark.core.settings_registry import (
+    CheckboxField,
+    CustomComponentField,
+    HeadingField,
+    MultiSelectField,
+    NumberField,
+    SelectField,
+    TableField,
+    register_on_save,
+    register_settings,
+)
+from shelfmark.core.request_policy import (
+    get_source_content_type_capabilities,
+    parse_policy_mode,
+    validate_policy_rules,
+)
+
+
+_REQUEST_DEFAULT_MODE_OPTIONS = [
+    {
+        "value": "download",
+        "label": "Download",
+        "description": "Everything can be downloaded directly.",
+    },
+    {
+        "value": "request_release",
+        "label": "Request Release",
+        "description": "Users must request a specific release.",
+    },
+    {
+        "value": "request_book",
+        "label": "Request Book",
+        "description": "Users request a book, admin picks the release.",
+    },
+    {
+        "value": "blocked",
+        "label": "Blocked",
+        "description": "No downloads or requests allowed.",
+    },
+]
+
+_REQUEST_MATRIX_MODE_OPTIONS = [
+    option for option in _REQUEST_DEFAULT_MODE_OPTIONS if option["value"] != "request_book"
+]
+
+_SELF_SETTINGS_SECTION_OPTIONS = [
+    {
+        "value": "delivery",
+        "label": "Delivery Preferences",
+        "description": "Show personal delivery output and destination settings.",
+    },
+    {
+        "value": "search",
+        "label": "Search Preferences",
+        "description": "Show personal search mode and provider settings.",
+    },
+    {
+        "value": "notifications",
+        "label": "Notifications",
+        "description": "Show personal notification route settings.",
+    },
+]
+_SELF_SETTINGS_SECTION_VALUES = {option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS}
+_SELF_SETTINGS_SECTION_DEFAULTS = [option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS]
+_SEARCH_MODE_VALUES = {"direct", "universal"}
+_SEARCH_PREFERENCE_PROVIDER_KEYS = {"METADATA_PROVIDER", "METADATA_PROVIDER_AUDIOBOOK"}
+_SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
+    "SEARCH_MODE",
+    "DEFAULT_RELEASE_SOURCE",
+    *_SEARCH_PREFERENCE_PROVIDER_KEYS,
+}
+
+_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE = {
+    "builtin": (
+        "Create and manage user accounts directly. Passwords are stored locally and users sign in "
+        "with their username and password."
+    ),
+    "oidc": (
+        "Users sign in through your identity provider. New accounts can be created automatically on "
+        "first login when auto-provisioning is enabled, or you can pre-create users here and they\u2019ll "
+        "be linked by email on first sign-in."
+    ),
+    "proxy": (
+        "Users are authenticated by your reverse proxy. Accounts are automatically created on first "
+        "sign-in. If a local user with a matching username already exists, it will be linked instead."
+    ),
+    "cwa": (
+        "User accounts are synced from your Calibre-Web database. Users are matched by email, and new "
+        "accounts are created here when new CWA users are found."
+    ),
+    "none": "Authentication is disabled. Anyone can access Shelfmark without signing in.",
+    "default": "Authentication is disabled. Anyone can access Shelfmark without signing in.",
+}
+
+
+def _get_request_source_options():
+    """Build request-policy source options from registered release sources."""
+    from shelfmark.release_sources import list_available_sources
+
+    options = []
+    for source in list_available_sources():
+        options.append(
+            {
+                "value": source["name"],
+                "label": source["display_name"],
+            }
+        )
+    return options
+
+
+def _get_request_policy_rule_columns():
+    source_capabilities = get_source_content_type_capabilities()
+    content_type_options = []
+
+    for source_name, supported_types in source_capabilities.items():
+        normalized_types = [t for t in ("ebook", "audiobook") if t in supported_types]
+        for content_type in normalized_types:
+            content_type_options.append(
+                {
+                    "value": content_type,
+                    "label": "Ebook" if content_type == "ebook" else "Audiobook",
+                    "childOf": source_name,
+                }
+            )
+
+    return [
+        {
+            "key": "source",
+            "label": "Source",
+            "type": "select",
+            "options": _get_request_source_options(),
+            "defaultValue": "",
+            "placeholder": "Select source...",
+        },
+        {
+            "key": "content_type",
+            "label": "Content Type",
+            "type": "select",
+            "options": content_type_options,
+            "defaultValue": "",
+            "placeholder": "Select content type...",
+            "filterByField": "source",
+        },
+        {
+            "key": "mode",
+            "label": "Mode",
+            "type": "select",
+            "options": _REQUEST_MATRIX_MODE_OPTIONS,
+            "defaultValue": "",
+            "placeholder": "Select mode...",
+        },
+    ]
+
+
+def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | None]:
+    """Validate and normalize a search preference value for user overrides."""
+    if key not in _SEARCH_PREFERENCE_VALIDATABLE_KEYS:
+        return value, None
+
+    if value is None:
+        return None, None
+
+    normalized_value = str(value).strip()
+
+    if key == "SEARCH_MODE":
+        normalized_mode = normalized_value.lower()
+        if normalized_mode not in _SEARCH_MODE_VALUES:
+            return value, "SEARCH_MODE must be 'direct' or 'universal'"
+        return normalized_mode, None
+
+    if key in _SEARCH_PREFERENCE_PROVIDER_KEYS:
+        if normalized_value == "":
+            return "", None
+        from shelfmark.metadata_providers import is_provider_registered
+
+        if not is_provider_registered(normalized_value):
+            return (
+                value,
+                f"{key} must be a valid metadata provider name or empty",
+            )
+        return normalized_value, None
+
+    if key == "DEFAULT_RELEASE_SOURCE":
+        if normalized_value == "":
+            return "", None
+        from shelfmark.release_sources import list_available_sources
+
+        valid_sources = {source["name"] for source in list_available_sources()}
+        if normalized_value not in valid_sources:
+            return (
+                value,
+                "DEFAULT_RELEASE_SOURCE must be a valid release source name or empty",
+            )
+        return normalized_value, None
+
+    return value, None
+
+
+def _on_save_users(values):
+    """Validate users/request-policy settings before persistence."""
+    if "VISIBLE_SELF_SETTINGS_SECTIONS" in values:
+        raw_sections = values["VISIBLE_SELF_SETTINGS_SECTIONS"]
+        if raw_sections is None:
+            candidate_sections: list[str] = []
+        elif isinstance(raw_sections, str):
+            candidate_sections = [s.strip() for s in raw_sections.split(",") if s.strip()]
+        elif isinstance(raw_sections, (list, tuple, set)):
+            candidate_sections = [str(section).strip() for section in raw_sections if str(section).strip()]
+        else:
+            return {
+                "error": True,
+                "message": "VISIBLE_SELF_SETTINGS_SECTIONS must be a list of section identifiers",
+                "values": values,
+            }
+
+        normalized_sections: list[str] = []
+        for section in candidate_sections:
+            if section not in _SELF_SETTINGS_SECTION_VALUES:
+                allowed = ", ".join(sorted(_SELF_SETTINGS_SECTION_VALUES))
+                return {
+                    "error": True,
+                    "message": (
+                        "VISIBLE_SELF_SETTINGS_SECTIONS contains an unsupported section "
+                        f"'{section}'. Supported values: {allowed}"
+                    ),
+                    "values": values,
+                }
+            if section not in normalized_sections:
+                normalized_sections.append(section)
+
+        values["VISIBLE_SELF_SETTINGS_SECTIONS"] = normalized_sections
+
+    if "REQUEST_POLICY_DEFAULT_EBOOK" in values:
+        if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_EBOOK"]) is None:
+            return {
+                "error": True,
+                "message": "REQUEST_POLICY_DEFAULT_EBOOK must be a valid policy mode",
+                "values": values,
+            }
+
+    if "REQUEST_POLICY_DEFAULT_AUDIOBOOK" in values:
+        if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_AUDIOBOOK"]) is None:
+            return {
+                "error": True,
+                "message": "REQUEST_POLICY_DEFAULT_AUDIOBOOK must be a valid policy mode",
+                "values": values,
+            }
+
+    if "REQUEST_POLICY_RULES" in values:
+        normalized_rules, errors = validate_policy_rules(values["REQUEST_POLICY_RULES"])
+        if errors:
+            return {
+                "error": True,
+                "message": "; ".join(errors),
+                "values": values,
+            }
+        values["REQUEST_POLICY_RULES"] = normalized_rules
+
+    for key in _SEARCH_PREFERENCE_VALIDATABLE_KEYS:
+        if key not in values:
+            continue
+        normalized_value, validation_error = validate_search_preference_value(key, values[key])
+        if validation_error:
+            return {
+                "error": True,
+                "message": validation_error,
+                "values": values,
+            }
+        values[key] = normalized_value
+
+    return {"error": False, "values": values}
+
+
+register_on_save("users", _on_save_users)
+
+
+@register_settings("users", "Users & Requests", icon="users", order=6)
+def users_settings():
+    """User management tab - rendered as a custom component on the frontend."""
+    return [
+        HeadingField(
+            key="users_heading",
+            title="Users",
+            description=_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE["default"],
+            description_by_auth_mode=_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE,
+        ),
+        CustomComponentField(
+            key="users_management",
+            component="users_management",
+        ),
+        MultiSelectField(
+            key="VISIBLE_SELF_SETTINGS_SECTIONS",
+            label="Visible Self-Settings Sections",
+            description=(
+                "Choose which personal settings sections are shown in My Account for non-admin users."
+            ),
+            options=_SELF_SETTINGS_SECTION_OPTIONS,
+            default=_SELF_SETTINGS_SECTION_DEFAULTS,
+            variant="dropdown",
+            env_supported=False,
+        ),
+        HeadingField(
+            key="requests_heading",
+            title="Requests",
+            description=(
+                "Choose what users can download directly and what needs approval first."
+            ),
+        ),
+        CheckboxField(
+            key="REQUESTS_ENABLED",
+            label="Enable Requests",
+            description=(
+                "Turn this off to let everyone download directly without needing approval."
+            ),
+            default=False,
+            user_overridable=True,
+        ),
+        CustomComponentField(
+            key="request_policy_editor",
+            component="request_policy_grid",
+            label="Request Rules",
+            description=(
+                "Fine-tune access per source. Source rules can only be the same or more restrictive than the default above."
+            ),
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+            wrap_in_field_wrapper=True,
+            value_fields=[
+                SelectField(
+                    key="REQUEST_POLICY_DEFAULT_EBOOK",
+                    label="Default Ebook Mode",
+                    description=(
+                        "Sets the baseline for all ebook sources."
+                    ),
+                    options=_REQUEST_DEFAULT_MODE_OPTIONS,
+                    default="download",
+                    user_overridable=True,
+                ),
+                SelectField(
+                    key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
+                    label="Default Audiobook Mode",
+                    description=(
+                        "Sets the baseline for all audiobook sources."
+                    ),
+                    options=_REQUEST_DEFAULT_MODE_OPTIONS,
+                    default="download",
+                    user_overridable=True,
+                ),
+                TableField(
+                    key="REQUEST_POLICY_RULES",
+                    label="Request Rules",
+                    description=(
+                        "Fine-tune access per source. Source rules can only be the same or more restrictive than the default above."
+                    ),
+                    columns=_get_request_policy_rule_columns,
+                    default=[],
+                    add_label="Add Rule",
+                    empty_message="No request policy rules configured.",
+                    env_supported=False,
+                    user_overridable=True,
+                ),
+            ],
+        ),
+        NumberField(
+            key="MAX_PENDING_REQUESTS_PER_USER",
+            label="Max pending requests per user",
+            description="How many open requests a user can have at a time.",
+            default=20,
+            min_value=1,
+            max_value=1000,
+            user_overridable=True,
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+        ),
+        CheckboxField(
+            key="REQUESTS_ALLOW_NOTES",
+            label="Allow notes on requests",
+            description="Let users add a note when they submit a request.",
+            default=True,
+            user_overridable=True,
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+        ),
+    ]

shelfmark/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core module - shared models, queue, and utilities."""
+
+from shelfmark.core.models import QueueItem, SearchFilters, QueueStatus
+from shelfmark.core.queue import BookQueue, book_queue
+from shelfmark.core.logger import setup_logger

shelfmark/core/activity_routes.py

@@ -0,0 +1,700 @@
+"""Activity API routes (snapshot, dismiss, history)."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, NamedTuple
+
+from flask import Flask, jsonify, request, session
+
+from shelfmark.core.activity_view_state_service import (
+    ADMIN_VIEWER_SCOPE,
+    NOAUTH_VIEWER_SCOPE,
+    ActivityViewStateService,
+    user_viewer_scope,
+)
+from shelfmark.core.download_history_service import ACTIVE_DOWNLOAD_STATUS, DownloadHistoryService, VALID_TERMINAL_STATUSES
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import ACTIVE_QUEUE_STATUSES, QueueStatus, TERMINAL_QUEUE_STATUSES
+from shelfmark.core.request_validation import RequestStatus
+from shelfmark.core.request_helpers import (
+    emit_ws_event,
+    extract_release_source_id,
+    normalize_positive_int,
+    populate_request_usernames,
+)
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+
+
+def _require_authenticated(resolve_auth_mode: Callable[[], str]):
+    auth_mode = resolve_auth_mode()
+    if auth_mode == "none":
+        return None
+    if "user_id" not in session:
+        return jsonify({"error": "Unauthorized"}), 401
+    return None
+
+
+def _resolve_db_user_id(
+    require_in_auth_mode: bool = True,
+    *,
+    user_db: UserDB | None = None,
+):
+    raw_db_user_id = session.get("db_user_id")
+    if raw_db_user_id is None:
+        if not require_in_auth_mode:
+            return None, None
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+    try:
+        parsed_db_user_id = int(raw_db_user_id)
+    except (TypeError, ValueError):
+        if not require_in_auth_mode:
+            return None, None
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+
+    if parsed_db_user_id < 1:
+        if not require_in_auth_mode:
+            return None, None
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+
+    if user_db is not None:
+        try:
+            db_user = user_db.get_user(user_id=parsed_db_user_id)
+        except Exception as exc:
+            logger.warning("Failed to validate activity db identity %s: %s", parsed_db_user_id, exc)
+            db_user = None
+        if db_user is None:
+            if not require_in_auth_mode:
+                return None, None
+            return None, (
+                jsonify(
+                    {
+                        "error": "User identity unavailable for activity workflow",
+                        "code": "user_identity_unavailable",
+                    }
+                ),
+                403,
+            )
+
+    return parsed_db_user_id, None
+
+
+class _ActorContext(NamedTuple):
+    db_user_id: int | None
+    is_no_auth: bool
+    is_admin: bool
+    owner_scope: int | None
+    viewer_scope: str
+
+
+def _resolve_activity_actor(
+    *,
+    user_db: UserDB,
+    resolve_auth_mode: Callable[[], str],
+) -> tuple[_ActorContext | None, Any | None]:
+    """Resolve acting user identity for activity mutations.
+
+    Returns (actor, error_response). On success actor is non-None.
+    """
+    if resolve_auth_mode() == "none":
+        return _ActorContext(
+            db_user_id=None,
+            is_no_auth=True,
+            is_admin=True,
+            owner_scope=None,
+            viewer_scope=NOAUTH_VIEWER_SCOPE,
+        ), None
+
+    db_user_id, db_gate = _resolve_db_user_id(user_db=user_db)
+    if db_user_id is None:
+        return None, db_gate
+
+    is_admin = bool(session.get("is_admin"))
+    viewer_scope = ADMIN_VIEWER_SCOPE if is_admin else user_viewer_scope(db_user_id)
+    return _ActorContext(
+        db_user_id=db_user_id,
+        is_no_auth=False,
+        is_admin=is_admin,
+        owner_scope=None if is_admin else db_user_id,
+        viewer_scope=viewer_scope,
+    ), None
+
+
+def _activity_ws_room(actor: _ActorContext) -> str:
+    """Resolve the WebSocket room for activity events."""
+    if actor.is_no_auth or actor.is_admin:
+        return "admins"
+    if actor.db_user_id is not None:
+        return f"user_{actor.db_user_id}"
+    return "admins"
+
+
+def _check_item_ownership(actor: _ActorContext, row: dict[str, Any]) -> Any | None:
+    """Return a 403 response if the actor doesn't own the item, else None."""
+    if actor.is_admin:
+        return None
+    owner_user_id = normalize_positive_int(row.get("user_id"))
+    if owner_user_id != actor.db_user_id:
+        return jsonify({"error": "Forbidden"}), 403
+    return None
+
+
+def _check_terminal_download(row: dict[str, Any]) -> Any | None:
+    final_status = str(row.get("final_status") or "").strip().lower()
+    if final_status not in VALID_TERMINAL_STATUSES:
+        return jsonify({"error": "Only terminal downloads can be dismissed"}), 409
+    return None
+
+
+def _check_terminal_request(row: dict[str, Any]) -> Any | None:
+    if _request_terminal_status(row) is None:
+        return jsonify({"error": "Only terminal requests can be dismissed"}), 409
+    return None
+
+
+def _list_visible_requests(user_db: UserDB, *, is_admin: bool, db_user_id: int | None) -> list[dict[str, Any]]:
+    if is_admin:
+        request_rows = user_db.list_requests()
+        populate_request_usernames(request_rows, user_db)
+        return request_rows
+
+    if db_user_id is None:
+        return []
+    return user_db.list_requests(user_id=db_user_id)
+
+
+def _parse_item_key(item_key: Any, prefix: str) -> str | None:
+    """Extract the value after 'prefix:' from an item_key string."""
+    if not isinstance(item_key, str) or not item_key.startswith(f"{prefix}:"):
+        return None
+    value = item_key.split(":", 1)[1].strip()
+    return value or None
+
+
+_ALL_BUCKET_KEYS = (*ACTIVE_QUEUE_STATUSES, *TERMINAL_QUEUE_STATUSES)
+
+
+def _build_download_status_from_db(
+    *,
+    db_rows: list[dict[str, Any]],
+    queue_status: dict[str, dict[str, Any]],
+) -> dict[str, dict[str, Any]]:
+    """Build the download status dict from DB rows, overlaying live queue data.
+
+    Active DB rows are matched against the queue for live progress.
+    Terminal DB rows go directly into their final bucket.
+    Stale active rows (no queue entry) are treated as interrupted errors.
+    """
+    status: dict[str, dict[str, Any]] = {key: {} for key in _ALL_BUCKET_KEYS}
+
+    # Index queue items by task_id for fast lookup: task_id -> (bucket_key, payload)
+    queue_index: dict[str, tuple[str, dict[str, Any]]] = {}
+    for bucket_key in _ALL_BUCKET_KEYS:
+        bucket = queue_status.get(bucket_key)
+        if not isinstance(bucket, dict):
+            continue
+        for task_id, payload in bucket.items():
+            queue_index[str(task_id)] = (bucket_key, payload)
+
+    for row in db_rows:
+        task_id = str(row.get("task_id") or "").strip()
+        if not task_id:
+            continue
+
+        final_status = row.get("final_status")
+
+        if final_status == ACTIVE_DOWNLOAD_STATUS:
+            queue_entry = queue_index.pop(task_id, None)
+            if queue_entry is not None:
+                bucket_key, queue_payload = queue_entry
+                status[bucket_key][task_id] = queue_payload
+            else:
+                # Stale active row — no queue entry means it was interrupted
+                download_payload = DownloadHistoryService.to_download_payload(row)
+                download_payload["status_message"] = "Interrupted"
+                status[QueueStatus.ERROR][task_id] = download_payload
+        elif final_status in VALID_TERMINAL_STATUSES:
+            download_payload = DownloadHistoryService.to_download_payload(row)
+            # For complete/cancelled the saved status_message is a stale
+            # progress string (e.g. "Fetching download sources") — clear it
+            # so the frontend only shows its own status label.  Error rows
+            # keep theirs since the message describes the failure.
+            if final_status in ("complete", "cancelled"):
+                download_payload["status_message"] = None
+            status[final_status][task_id] = download_payload
+
+    return status
+
+
+def _request_terminal_status(row: dict[str, Any]) -> str | None:
+    request_status = row.get("status")
+    if request_status == RequestStatus.PENDING:
+        return None
+    if request_status == RequestStatus.REJECTED:
+        return RequestStatus.REJECTED
+    if request_status == RequestStatus.CANCELLED:
+        return RequestStatus.CANCELLED
+    if request_status != RequestStatus.FULFILLED:
+        return None
+
+    delivery_state = str(row.get("delivery_state") or "").strip().lower()
+    if delivery_state in {QueueStatus.ERROR, QueueStatus.CANCELLED}:
+        return delivery_state
+    return QueueStatus.COMPLETE
+
+
+def _minimal_request_snapshot(request_row: dict[str, Any], request_id: int) -> dict[str, Any]:
+    book_data = request_row.get("book_data")
+    release_data = request_row.get("release_data")
+    if not isinstance(book_data, dict):
+        book_data = {}
+    if not isinstance(release_data, dict):
+        release_data = {}
+
+    minimal_request = {
+        "id": request_id,
+        "user_id": request_row.get("user_id"),
+        "status": request_row.get("status"),
+        "request_level": request_row.get("request_level"),
+        "delivery_state": request_row.get("delivery_state"),
+        "book_data": book_data,
+        "release_data": release_data,
+        "note": request_row.get("note"),
+        "admin_note": request_row.get("admin_note"),
+        "created_at": request_row.get("created_at"),
+        "updated_at": request_row.get("reviewed_at") or request_row.get("created_at"),
+    }
+    username = request_row.get("username")
+    if isinstance(username, str):
+        minimal_request["username"] = username
+    return {"kind": "request", "request": minimal_request}
+
+
+def _request_history_entry(
+    request_row: dict[str, Any],
+    *,
+    dismissed_at: str | None,
+) -> dict[str, Any] | None:
+    request_id = normalize_positive_int(request_row.get("id"))
+    if request_id is None:
+        return None
+    final_status = _request_terminal_status(request_row)
+    item_key = f"request:{request_id}"
+    return {
+        "id": item_key,
+        "user_id": request_row.get("user_id"),
+        "item_type": "request",
+        "item_key": item_key,
+        "dismissed_at": dismissed_at,
+        "snapshot": _minimal_request_snapshot(request_row, request_id),
+        "origin": "request",
+        "final_status": final_status,
+        "terminal_at": request_row.get("reviewed_at") or request_row.get("created_at"),
+        "request_id": request_id,
+        "source_id": extract_release_source_id(request_row.get("release_data")),
+    }
+
+
+def register_activity_routes(
+    app: Flask,
+    user_db: UserDB,
+    *,
+    activity_view_state_service: ActivityViewStateService,
+    download_history_service: DownloadHistoryService,
+    resolve_auth_mode: Callable[[], str],
+    queue_status: Callable[..., dict[str, dict[str, Any]]],
+    sync_request_delivery_states: Callable[..., list[dict[str, Any]]],
+    emit_request_updates: Callable[[list[dict[str, Any]]], None],
+    ws_manager: Any | None = None,
+) -> None:
+    """Register activity routes."""
+
+    @app.route("/api/activity/snapshot", methods=["GET"])
+    def api_activity_snapshot():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        actor, actor_error = _resolve_activity_actor(
+            user_db=user_db,
+            resolve_auth_mode=resolve_auth_mode,
+        )
+        if actor_error is not None:
+            return actor_error
+
+        hidden_rows = activity_view_state_service.list_hidden(viewer_scope=actor.viewer_scope)
+        hidden_item_keys = {str(row.get("item_key") or "").strip() for row in hidden_rows}
+        dismissed_entries = [
+            {
+                "item_type": str(row.get("item_type") or "").strip().lower(),
+                "item_key": str(row.get("item_key") or "").strip(),
+            }
+            for row in hidden_rows
+            if str(row.get("item_type") or "").strip().lower() in {"download", "request"}
+            and str(row.get("item_key") or "").strip()
+        ]
+        live_queue = queue_status(user_id=actor.owner_scope)
+        db_rows = download_history_service.list_recent(
+            user_id=actor.owner_scope,
+            limit=200,
+        )
+        visible_db_rows = [
+            row
+            for row in db_rows
+            if f"download:{str(row.get('task_id') or '').strip()}" not in hidden_item_keys
+        ]
+
+        status = _build_download_status_from_db(
+            db_rows=visible_db_rows,
+            queue_status=live_queue,
+        )
+
+        updated_requests = sync_request_delivery_states(
+            user_db,
+            queue_status=status,
+            user_id=actor.owner_scope,
+        )
+        emit_request_updates(updated_requests)
+        request_rows = _list_visible_requests(
+            user_db,
+            is_admin=actor.is_admin,
+            db_user_id=actor.db_user_id,
+        )
+        visible_request_rows: list[dict[str, Any]] = []
+        for row in request_rows:
+            request_id = normalize_positive_int(row.get("id"))
+            if request_id is None:
+                continue
+            if f"request:{request_id}" in hidden_item_keys:
+                continue
+            visible_request_rows.append(row)
+
+        return jsonify(
+            {
+                "status": status,
+                "requests": visible_request_rows,
+                "dismissed": dismissed_entries,
+            }
+        )
+
+    @app.route("/api/activity/dismiss", methods=["POST"])
+    def api_activity_dismiss():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        actor, actor_error = _resolve_activity_actor(
+            user_db=user_db,
+            resolve_auth_mode=resolve_auth_mode,
+        )
+        if actor_error is not None:
+            return actor_error
+
+        data = request.get_json(silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": "Invalid payload"}), 400
+
+        item_type = str(data.get("item_type") or "").strip().lower()
+        item_key = data.get("item_key")
+
+        dismissal_item: dict[str, str] | None = None
+
+        if item_type == "download":
+            task_id = _parse_item_key(item_key, "download")
+            if task_id is None:
+                return jsonify({"error": "item_key must be in the format download:<task_id>"}), 400
+
+            existing = download_history_service.get_by_task_id(task_id)
+            if existing is None:
+                return jsonify({"error": "Download not found"}), 404
+
+            ownership_gate = _check_item_ownership(actor, existing)
+            if ownership_gate is not None:
+                return ownership_gate
+            terminal_gate = _check_terminal_download(existing)
+            if terminal_gate is not None:
+                return terminal_gate
+
+            activity_view_state_service.dismiss(
+                viewer_scope=actor.viewer_scope,
+                item_type="download",
+                item_key=f"download:{task_id}",
+            )
+            dismissal_item = {"item_type": "download", "item_key": f"download:{task_id}"}
+
+        elif item_type == "request":
+            request_id = normalize_positive_int(_parse_item_key(item_key, "request"))
+            if request_id is None:
+                return jsonify({"error": "item_key must be in the format request:<id>"}), 400
+
+            request_row = user_db.get_request(request_id)
+            if request_row is None:
+                return jsonify({"error": "Request not found"}), 404
+
+            ownership_gate = _check_item_ownership(actor, request_row)
+            if ownership_gate is not None:
+                return ownership_gate
+            terminal_gate = _check_terminal_request(request_row)
+            if terminal_gate is not None:
+                return terminal_gate
+
+            activity_view_state_service.dismiss(
+                viewer_scope=actor.viewer_scope,
+                item_type="request",
+                item_key=f"request:{request_id}",
+            )
+            dismissal_item = {"item_type": "request", "item_key": f"request:{request_id}"}
+        else:
+            return jsonify({"error": "item_type must be one of: download, request"}), 400
+
+        room = _activity_ws_room(actor)
+        emit_ws_event(
+            ws_manager,
+            event_name="activity_update",
+            room=room,
+            payload={
+                "kind": "dismiss",
+                "item_type": dismissal_item["item_type"],
+                "item_key": dismissal_item["item_key"],
+            },
+        )
+
+        return jsonify({"status": "dismissed", "item": dismissal_item})
+
+    @app.route("/api/activity/dismiss-many", methods=["POST"])
+    def api_activity_dismiss_many():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        actor, actor_error = _resolve_activity_actor(
+            user_db=user_db,
+            resolve_auth_mode=resolve_auth_mode,
+        )
+        if actor_error is not None:
+            return actor_error
+
+        data = request.get_json(silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": "Invalid payload"}), 400
+        items = data.get("items")
+        if not isinstance(items, list):
+            return jsonify({"error": "items must be an array"}), 400
+
+        dismissal_items: list[dict[str, str]] = []
+        missing_item_keys: list[str] = []
+
+        for item in items:
+            if not isinstance(item, dict):
+                return jsonify({"error": "items must contain objects"}), 400
+
+            item_type = str(item.get("item_type") or "").strip().lower()
+            item_key = item.get("item_key")
+
+            if item_type == "download":
+                task_id = _parse_item_key(item_key, "download")
+                if task_id is None:
+                    return jsonify({"error": "download item_key must be in the format download:<task_id>"}), 400
+                existing = download_history_service.get_by_task_id(task_id)
+                if existing is None:
+                    missing_item_keys.append(f"download:{task_id}")
+                    continue
+                ownership_gate = _check_item_ownership(actor, existing)
+                if ownership_gate is not None:
+                    return ownership_gate
+                terminal_gate = _check_terminal_download(existing)
+                if terminal_gate is not None:
+                    return terminal_gate
+                dismissal_items.append({"item_type": "download", "item_key": f"download:{task_id}"})
+                continue
+
+            if item_type == "request":
+                request_id = normalize_positive_int(_parse_item_key(item_key, "request"))
+                if request_id is None:
+                    return jsonify({"error": "request item_key must be in the format request:<id>"}), 400
+                request_row = user_db.get_request(request_id)
+                if request_row is None:
+                    missing_item_keys.append(f"request:{request_id}")
+                    continue
+                ownership_gate = _check_item_ownership(actor, request_row)
+                if ownership_gate is not None:
+                    return ownership_gate
+                terminal_gate = _check_terminal_request(request_row)
+                if terminal_gate is not None:
+                    return terminal_gate
+                dismissal_items.append({"item_type": "request", "item_key": f"request:{request_id}"})
+                continue
+
+            return jsonify({"error": "item_type must be one of: download, request"}), 400
+
+        if missing_item_keys:
+            return (
+                jsonify(
+                    {
+                        "error": "One or more activity items were not found",
+                        "missing_item_keys": missing_item_keys,
+                    }
+                ),
+                404,
+            )
+
+        dismissed_count = activity_view_state_service.dismiss_many(
+            viewer_scope=actor.viewer_scope,
+            items=dismissal_items,
+        )
+
+        room = _activity_ws_room(actor)
+        emit_ws_event(
+            ws_manager,
+            event_name="activity_update",
+            room=room,
+            payload={
+                "kind": "dismiss_many",
+                "count": dismissed_count,
+            },
+        )
+
+        return jsonify({"status": "dismissed", "count": dismissed_count})
+
+    @app.route("/api/activity/history", methods=["GET"])
+    def api_activity_history():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        actor, actor_error = _resolve_activity_actor(
+            user_db=user_db,
+            resolve_auth_mode=resolve_auth_mode,
+        )
+        if actor_error is not None:
+            return actor_error
+
+        limit = request.args.get("limit", type=int, default=50)
+        offset = request.args.get("offset", type=int, default=0)
+        if limit is None:
+            limit = 50
+        if offset is None:
+            offset = 0
+        if limit < 1:
+            return jsonify({"error": "limit must be a positive integer"}), 400
+        if offset < 0:
+            return jsonify({"error": "offset must be a non-negative integer"}), 400
+
+        history_rows = activity_view_state_service.list_history(
+            viewer_scope=actor.viewer_scope,
+            limit=limit,
+            offset=offset,
+        )
+        payload: list[dict[str, Any]] = []
+
+        for history_row in history_rows:
+            item_type = str(history_row.get("item_type") or "").strip().lower()
+            item_key = str(history_row.get("item_key") or "").strip()
+            dismissed_at = history_row.get("dismissed_at")
+
+            if not isinstance(dismissed_at, str) or not dismissed_at.strip():
+                raise RuntimeError(f"Activity history state missing dismissed_at for {item_key}")
+
+            if item_type == "download":
+                task_id = _parse_item_key(item_key, "download")
+                if task_id is None:
+                    raise RuntimeError(f"Invalid activity history item_key: {item_key}")
+
+                download_row = download_history_service.get_by_task_id(task_id)
+                if download_row is None:
+                    raise RuntimeError(f"Download history row not found for {item_key}")
+
+                if not actor.is_admin:
+                    owner_user_id = normalize_positive_int(download_row.get("user_id"))
+                    if owner_user_id != actor.db_user_id:
+                        raise RuntimeError(f"Viewer state out of scope for {item_key}")
+
+                payload.append(
+                    DownloadHistoryService.to_history_row(
+                        download_row,
+                        dismissed_at=dismissed_at,
+                    )
+                )
+                continue
+
+            if item_type == "request":
+                request_id = normalize_positive_int(_parse_item_key(item_key, "request"))
+                if request_id is None:
+                    raise RuntimeError(f"Invalid activity history item_key: {item_key}")
+
+                request_row = user_db.get_request(request_id)
+                if request_row is None:
+                    raise RuntimeError(f"Request row not found for {item_key}")
+
+                if not actor.is_admin:
+                    owner_user_id = normalize_positive_int(request_row.get("user_id"))
+                    if owner_user_id != actor.db_user_id:
+                        raise RuntimeError(f"Viewer state out of scope for {item_key}")
+
+                populate_request_usernames([request_row], user_db)
+                entry = _request_history_entry(
+                    request_row,
+                    dismissed_at=dismissed_at,
+                )
+                if entry is None:
+                    raise RuntimeError(f"Failed to build request history entry for {item_key}")
+                payload.append(entry)
+                continue
+
+            raise RuntimeError(f"Unknown activity history item_type: {item_type}")
+
+        return jsonify(payload)
+
+    @app.route("/api/activity/history", methods=["DELETE"])
+    def api_activity_history_clear():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        actor, actor_error = _resolve_activity_actor(
+            user_db=user_db,
+            resolve_auth_mode=resolve_auth_mode,
+        )
+        if actor_error is not None:
+            return actor_error
+
+        cleared_count = activity_view_state_service.clear_history(
+            viewer_scope=actor.viewer_scope,
+        )
+
+        room = _activity_ws_room(actor)
+        emit_ws_event(
+            ws_manager,
+            event_name="activity_update",
+            room=room,
+            payload={
+                "kind": "history_cleared",
+                "count": cleared_count,
+            },
+        )
+        return jsonify({"status": "cleared", "cleared_count": cleared_count})

shelfmark/core/activity_view_state_service.py

@@ -0,0 +1,310 @@
+"""Persistence helpers for per-viewer activity visibility state."""
+
+from __future__ import annotations
+
+import sqlite3
+import threading
+from typing import Any
+
+from shelfmark.core.request_helpers import now_utc_iso
+
+
+VALID_ACTIVITY_ITEM_TYPES = frozenset({"download", "request"})
+ADMIN_VIEWER_SCOPE = "admin:shared"
+NOAUTH_VIEWER_SCOPE = "noauth:shared"
+USER_VIEWER_SCOPE_PREFIX = "user:"
+
+
+def user_viewer_scope(user_id: int) -> str:
+    if not isinstance(user_id, int) or user_id < 1:
+        raise ValueError("user_id must be a positive integer")
+    return f"{USER_VIEWER_SCOPE_PREFIX}{user_id}"
+
+
+def normalize_viewer_scope(viewer_scope: Any) -> str:
+    if not isinstance(viewer_scope, str) or not viewer_scope.strip():
+        raise ValueError("viewer_scope must be a non-empty string")
+
+    normalized = viewer_scope.strip()
+    if normalized in {ADMIN_VIEWER_SCOPE, NOAUTH_VIEWER_SCOPE}:
+        return normalized
+
+    if not normalized.startswith(USER_VIEWER_SCOPE_PREFIX):
+        raise ValueError(
+            "viewer_scope must be one of: admin:shared, noauth:shared, or user:<id>"
+        )
+
+    raw_user_id = normalized[len(USER_VIEWER_SCOPE_PREFIX):].strip()
+    try:
+        parsed_user_id = int(raw_user_id)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("viewer_scope user id must be a positive integer") from exc
+
+    return user_viewer_scope(parsed_user_id)
+
+
+def _normalize_item_type(item_type: Any) -> str:
+    if not isinstance(item_type, str) or not item_type.strip():
+        raise ValueError("item_type must be a non-empty string")
+    normalized = item_type.strip().lower()
+    if normalized not in VALID_ACTIVITY_ITEM_TYPES:
+        raise ValueError("item_type must be one of: download, request")
+    return normalized
+
+
+def _normalize_item_key(item_key: Any, *, item_type: str) -> str:
+    if not isinstance(item_key, str) or not item_key.strip():
+        raise ValueError("item_key must be a non-empty string")
+
+    normalized = item_key.strip()
+    expected_prefix = f"{item_type}:"
+    if not normalized.startswith(expected_prefix):
+        raise ValueError(f"item_key must be in the format {expected_prefix}<id>")
+    if not normalized.split(":", 1)[1].strip():
+        raise ValueError(f"item_key must be in the format {expected_prefix}<id>")
+    return normalized
+
+
+class ActivityViewStateService:
+    """Service for per-viewer activity dismissal and history visibility."""
+
+    def __init__(self, db_path: str):
+        self._db_path = db_path
+        self._lock = threading.Lock()
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA foreign_keys = ON")
+        return conn
+
+    def list_hidden(
+        self,
+        *,
+        viewer_scope: str,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        normalized_scope = normalize_viewer_scope(viewer_scope)
+        normalized_limit = None if limit is None else max(1, int(limit))
+        query = """
+            SELECT item_type, item_key, dismissed_at, cleared_at
+            FROM activity_view_state
+            WHERE viewer_scope = ?
+              AND dismissed_at IS NOT NULL
+            ORDER BY COALESCE(cleared_at, dismissed_at) DESC, id DESC
+        """
+        params: list[Any] = [normalized_scope]
+        if normalized_limit is not None:
+            query += "\nLIMIT ?"
+            params.append(normalized_limit)
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(query, params).fetchall()
+            return [dict(row) for row in rows]
+        finally:
+            conn.close()
+
+    def list_history(
+        self,
+        *,
+        viewer_scope: str,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> list[dict[str, Any]]:
+        normalized_scope = normalize_viewer_scope(viewer_scope)
+        normalized_limit = max(1, min(int(limit), 5000))
+        normalized_offset = max(0, int(offset))
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT item_type, item_key, dismissed_at
+                FROM activity_view_state
+                WHERE viewer_scope = ?
+                  AND dismissed_at IS NOT NULL
+                  AND cleared_at IS NULL
+                ORDER BY dismissed_at DESC, id DESC
+                LIMIT ? OFFSET ?
+                """,
+                (normalized_scope, normalized_limit, normalized_offset),
+            ).fetchall()
+            return [dict(row) for row in rows]
+        finally:
+            conn.close()
+
+    def dismiss(
+        self,
+        *,
+        viewer_scope: str,
+        item_type: str,
+        item_key: str,
+    ) -> int:
+        normalized_scope = normalize_viewer_scope(viewer_scope)
+        normalized_type = _normalize_item_type(item_type)
+        normalized_key = _normalize_item_key(item_key, item_type=normalized_type)
+        dismissed_at = now_utc_iso()
+
+        with self._lock:
+            conn = self._connect()
+            try:
+                cursor = conn.execute(
+                    """
+                    INSERT INTO activity_view_state (
+                        viewer_scope,
+                        item_type,
+                        item_key,
+                        dismissed_at,
+                        cleared_at
+                    )
+                    VALUES (?, ?, ?, ?, NULL)
+                    ON CONFLICT(viewer_scope, item_type, item_key) DO UPDATE SET
+                        dismissed_at = excluded.dismissed_at,
+                        cleared_at = NULL
+                    """,
+                    (normalized_scope, normalized_type, normalized_key, dismissed_at),
+                )
+                conn.commit()
+                rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                return max(rowcount, 0)
+            finally:
+                conn.close()
+
+    def dismiss_many(
+        self,
+        *,
+        viewer_scope: str,
+        items: list[dict[str, str]],
+    ) -> int:
+        normalized_scope = normalize_viewer_scope(viewer_scope)
+        if not items:
+            return 0
+
+        seen: set[tuple[str, str]] = set()
+        normalized_items: list[tuple[str, str]] = []
+        for item in items:
+            normalized_type = _normalize_item_type(item.get("item_type"))
+            normalized_key = _normalize_item_key(item.get("item_key"), item_type=normalized_type)
+            marker = (normalized_type, normalized_key)
+            if marker in seen:
+                continue
+            seen.add(marker)
+            normalized_items.append(marker)
+
+        if not normalized_items:
+            return 0
+
+        dismissed_at = now_utc_iso()
+        with self._lock:
+            conn = self._connect()
+            try:
+                total = 0
+                for normalized_type, normalized_key in normalized_items:
+                    cursor = conn.execute(
+                        """
+                        INSERT INTO activity_view_state (
+                            viewer_scope,
+                            item_type,
+                            item_key,
+                            dismissed_at,
+                            cleared_at
+                        )
+                        VALUES (?, ?, ?, ?, NULL)
+                        ON CONFLICT(viewer_scope, item_type, item_key) DO UPDATE SET
+                            dismissed_at = excluded.dismissed_at,
+                            cleared_at = NULL
+                        """,
+                        (normalized_scope, normalized_type, normalized_key, dismissed_at),
+                    )
+                    rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                    total += max(rowcount, 0)
+                conn.commit()
+                return total
+            finally:
+                conn.close()
+
+    def clear_history(self, *, viewer_scope: str) -> int:
+        normalized_scope = normalize_viewer_scope(viewer_scope)
+        cleared_at = now_utc_iso()
+
+        with self._lock:
+            conn = self._connect()
+            try:
+                cursor = conn.execute(
+                    """
+                    UPDATE activity_view_state
+                    SET cleared_at = ?
+                    WHERE viewer_scope = ?
+                      AND dismissed_at IS NOT NULL
+                      AND cleared_at IS NULL
+                    """,
+                    (cleared_at, normalized_scope),
+                )
+                conn.commit()
+                rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                return max(rowcount, 0)
+            finally:
+                conn.close()
+
+    def clear_item_for_all_viewers(self, *, item_type: str, item_key: str) -> int:
+        normalized_type = _normalize_item_type(item_type)
+        normalized_key = _normalize_item_key(item_key, item_type=normalized_type)
+
+        with self._lock:
+            conn = self._connect()
+            try:
+                cursor = conn.execute(
+                    """
+                    DELETE FROM activity_view_state
+                    WHERE item_type = ? AND item_key = ?
+                    """,
+                    (normalized_type, normalized_key),
+                )
+                conn.commit()
+                rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                return max(rowcount, 0)
+            finally:
+                conn.close()
+
+    def delete_viewer_scope(self, *, viewer_scope: str) -> int:
+        normalized_scope = normalize_viewer_scope(viewer_scope)
+
+        with self._lock:
+            conn = self._connect()
+            try:
+                cursor = conn.execute(
+                    "DELETE FROM activity_view_state WHERE viewer_scope = ?",
+                    (normalized_scope,),
+                )
+                conn.commit()
+                rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                return max(rowcount, 0)
+            finally:
+                conn.close()
+
+    def delete_items(self, *, item_type: str, item_keys: list[str]) -> int:
+        normalized_type = _normalize_item_type(item_type)
+        normalized_keys = [
+            _normalize_item_key(item_key, item_type=normalized_type)
+            for item_key in item_keys
+        ]
+        if not normalized_keys:
+            return 0
+
+        placeholders = ",".join("?" for _ in normalized_keys)
+        with self._lock:
+            conn = self._connect()
+            try:
+                cursor = conn.execute(
+                    f"""
+                    DELETE FROM activity_view_state
+                    WHERE item_type = ? AND item_key IN ({placeholders})
+                    """,
+                    (normalized_type, *normalized_keys),
+                )
+                conn.commit()
+                rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                return max(rowcount, 0)
+            finally:
+                conn.close()

shelfmark/core/admin_routes.py

@@ -0,0 +1,420 @@
+"""Admin user management API routes.
+
+Registers /api/admin/users CRUD endpoints for managing users.
+All endpoints require admin session.
+"""
+
+from functools import wraps
+import os
+import sqlite3
+from typing import Any
+
+from flask import Flask, g, jsonify, request, session
+from werkzeug.security import generate_password_hash
+
+from shelfmark.config.booklore_settings import (
+    get_booklore_library_options,
+    get_booklore_path_options,
+)
+from shelfmark.config.env import CWA_DB_PATH
+from shelfmark.core.admin_settings_routes import (
+    register_admin_settings_routes,
+    validate_user_settings,
+)
+from shelfmark.core.auth_modes import (
+    AUTH_SOURCE_BUILTIN,
+    AUTH_SOURCE_CWA,
+    AUTH_SOURCE_OIDC,
+    AUTH_SOURCE_PROXY,
+    is_user_active_for_auth_mode,
+    load_active_auth_mode,
+    normalize_auth_source,
+)
+from shelfmark.core.cwa_user_sync import sync_cwa_users_from_rows
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+
+
+def _get_user_edit_capabilities(
+    user: dict[str, Any],
+    security_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Return backend-authored capability flags for the user edit form."""
+    auth_source = normalize_auth_source(
+        user.get("auth_source"),
+        user.get("oidc_subject"),
+    )
+    if security_config is None and auth_source == AUTH_SOURCE_OIDC:
+        security_config = load_config_file("security")
+
+    oidc_use_admin_group = bool((security_config or {}).get("OIDC_USE_ADMIN_GROUP", True))
+    role_managed_by_oidc_group = auth_source == AUTH_SOURCE_OIDC and oidc_use_admin_group
+    can_edit_role = auth_source == AUTH_SOURCE_BUILTIN or (
+        auth_source == AUTH_SOURCE_OIDC and not role_managed_by_oidc_group
+    )
+
+    return {
+        "authSource": auth_source,
+        "canSetPassword": auth_source == AUTH_SOURCE_BUILTIN,
+        "canEditRole": can_edit_role,
+        "canEditEmail": auth_source in {AUTH_SOURCE_BUILTIN, AUTH_SOURCE_PROXY},
+        "canEditDisplayName": auth_source != AUTH_SOURCE_OIDC,
+    }
+
+
+def _sanitize_user(user: dict) -> dict:
+    """Remove sensitive fields from user dict before returning to client."""
+    sanitized = dict(user)
+    sanitized.pop("password_hash", None)
+    return sanitized
+
+
+def _oidc_role_management_message(security_config: dict[str, Any]) -> str:
+    admin_group = security_config.get("OIDC_ADMIN_GROUP", "")
+    if admin_group:
+        return (
+            "Admin roles for OIDC users are managed by the "
+            f"'{admin_group}' group in your identity provider"
+        )
+    return (
+        "Disable 'Use Admin Group for Authorization' in security settings "
+        "to manage roles manually"
+    )
+
+
+def _serialize_user(
+    user: dict[str, Any],
+    auth_method: str,
+    security_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Sanitize and enrich a user payload for API responses."""
+    payload = _sanitize_user(user)
+    payload["auth_source"] = normalize_auth_source(
+        payload.get("auth_source"),
+        payload.get("oidc_subject"),
+    )
+    payload["is_active"] = is_user_active_for_auth_mode(payload, auth_method)
+    payload["edit_capabilities"] = _get_user_edit_capabilities(
+        payload,
+        security_config=security_config,
+    )
+    return payload
+
+
+
+
+def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]:
+    """Sync all users from the Calibre-Web database into users.db."""
+    if not CWA_DB_PATH or not CWA_DB_PATH.exists():
+        raise FileNotFoundError("Calibre-Web database is not available")
+
+    db_path = os.fspath(CWA_DB_PATH)
+    db_uri = f"file:{db_path}?mode=ro&immutable=1"
+    conn = sqlite3.connect(db_uri, uri=True)
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT name, role, email FROM user")
+        rows = cur.fetchall()
+    finally:
+        conn.close()
+
+    return sync_cwa_users_from_rows(user_db, rows)
+
+
+def register_admin_routes(app: Flask, user_db: UserDB) -> None:
+    """Register admin user management routes on the Flask app."""
+
+    def _require_admin(f):
+        """Decorator to require admin session for admin routes.
+
+        In no-auth mode, everyone has access (is_admin defaults True).
+        In auth-required modes, requires an authenticated session with admin role.
+        Caches the resolved auth_mode in ``g.auth_mode`` for the request.
+        """
+        @wraps(f)
+        def decorated(*args, **kwargs):
+            auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db)
+            g.auth_mode = auth_mode
+            if auth_mode != "none":
+                if "user_id" not in session:
+                    return jsonify({"error": "Authentication required"}), 401
+                if not session.get("is_admin", False):
+                    return jsonify({"error": "Admin access required"}), 403
+            return f(*args, **kwargs)
+        return decorated
+
+    @app.route("/api/admin/users", methods=["GET"])
+    @_require_admin
+    def admin_list_users():
+        """List all users."""
+        users = user_db.list_users()
+        auth_mode = g.auth_mode
+        security_config = load_config_file("security")
+        return jsonify([
+            _serialize_user(u, auth_mode, security_config=security_config)
+            for u in users
+        ])
+
+    @app.route("/api/admin/users", methods=["POST"])
+    @_require_admin
+    def admin_create_user():
+        """Create a new user with password authentication."""
+        data = request.get_json() or {}
+        auth_mode = g.auth_mode
+
+        username = (data.get("username") or "").strip()
+        password = data.get("password", "")
+        email = (data.get("email") or "").strip() or None
+        display_name = (data.get("display_name") or "").strip() or None
+        role = data.get("role", "user")
+
+        if auth_mode in {AUTH_SOURCE_PROXY, AUTH_SOURCE_CWA}:
+            return jsonify({
+                "error": "Local user creation is disabled in this authentication mode",
+                "message": (
+                    "Users are provisioned by your external authentication source. "
+                    "Switch to builtin or OIDC mode to create local users."
+                ),
+            }), 400
+
+        if not username:
+            return jsonify({"error": "Username is required"}), 400
+        if not password or len(password) < 4:
+            return jsonify({"error": "Password must be at least 4 characters"}), 400
+        if role not in ("admin", "user"):
+            return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
+
+        # First user is always admin
+        existing_users = user_db.list_users()
+        if not existing_users:
+            role = "admin"
+
+        # Check if username already exists
+        if user_db.get_user(username=username):
+            return jsonify({"error": "Username already exists"}), 409
+
+        password_hash = generate_password_hash(password)
+        try:
+            user = user_db.create_user(
+                username=username,
+                password_hash=password_hash,
+                email=email,
+                display_name=display_name,
+                auth_source=AUTH_SOURCE_BUILTIN,
+                role=role,
+            )
+        except ValueError:
+            return jsonify({"error": "Username already exists"}), 409
+        logger.info(
+            "Shelfmark user created "
+            f"(source=manual_admin_create, created_by={session.get('user_id', 'unknown')}, "
+            f"username={username}, role={role}, auth_source={AUTH_SOURCE_BUILTIN})"
+        )
+        return jsonify(
+            _serialize_user(
+                user,
+                g.auth_mode,
+                security_config=load_config_file("security"),
+            )
+        ), 201
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["GET"])
+    @_require_admin
+    def admin_get_user(user_id):
+        """Get a user by ID with their settings."""
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        result = _serialize_user(
+            user,
+            g.auth_mode,
+            security_config=load_config_file("security"),
+        )
+        result["settings"] = user_db.get_user_settings(user_id)
+        return jsonify(result)
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["PUT"])
+    @_require_admin
+    def admin_update_user(user_id):
+        """Update user fields and/or settings."""
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        data = request.get_json() or {}
+        security_config = load_config_file("security")
+        auth_source = normalize_auth_source(
+            user.get("auth_source"),
+            user.get("oidc_subject"),
+        )
+        capabilities = _get_user_edit_capabilities(user, security_config=security_config)
+
+        # Handle optional password update
+        password = data.get("password", "")
+        if password:
+            if not capabilities["canSetPassword"]:
+                return jsonify({
+                    "error": f"Cannot set password for {auth_source.upper()} users",
+                    "message": "Password authentication is only available for local users.",
+                }), 400
+            if len(password) < 4:
+                return jsonify({"error": "Password must be at least 4 characters"}), 400
+            user_db.update_user(user_id, password_hash=generate_password_hash(password))
+
+        # Update user fields
+        user_fields = {}
+        for field in ("role", "email", "display_name"):
+            if field in data:
+                user_fields[field] = data[field]
+
+        if "role" in user_fields and user_fields["role"] not in ("admin", "user"):
+            return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
+
+        role_changed = "role" in user_fields and user_fields["role"] != user.get("role")
+        email_changed = "email" in user_fields and user_fields["email"] != user.get("email")
+        display_name_changed = (
+            "display_name" in user_fields
+            and user_fields["display_name"] != user.get("display_name")
+        )
+
+        if role_changed and not capabilities["canEditRole"]:
+            if auth_source == AUTH_SOURCE_OIDC:
+                return jsonify({
+                    "error": "Cannot change role for OIDC user when group-based authorization is enabled",
+                    "message": _oidc_role_management_message(security_config),
+                }), 400
+
+            return jsonify({
+                "error": f"Cannot change role for {auth_source.upper()} users",
+                "message": "Role is managed by the external authentication source.",
+            }), 400
+
+        if email_changed and not capabilities["canEditEmail"]:
+            if auth_source == AUTH_SOURCE_CWA:
+                return jsonify({
+                    "error": "Cannot change email for CWA users",
+                    "message": "Email is synced from Calibre-Web.",
+                }), 400
+
+            return jsonify({
+                "error": "Cannot change email for OIDC users",
+                "message": "Email is managed by your identity provider.",
+            }), 400
+
+        if display_name_changed and not capabilities["canEditDisplayName"]:
+            return jsonify({
+                "error": "Cannot change display name for OIDC users",
+                "message": "Display name is managed by your identity provider.",
+            }), 400
+
+        # Allow demoting the last admin account.
+        # Auth mode resolution automatically falls back to "none" when no
+        # local password admin remains.
+
+        # Avoid unnecessary writes for no-op field updates.
+        for field in ("role", "email", "display_name"):
+            if field in user_fields and user_fields[field] == user.get(field):
+                user_fields.pop(field)
+
+        if user_fields:
+            user_db.update_user(user_id, **user_fields)
+
+        # Update per-user settings
+        if "settings" in data:
+            if not isinstance(data["settings"], dict):
+                return jsonify({"error": "Settings must be an object"}), 400
+
+            validated_settings, validation_errors = validate_user_settings(data["settings"])
+            if validation_errors:
+                return jsonify({
+                    "error": "Invalid settings payload",
+                    "details": validation_errors,
+                }), 400
+
+            user_db.set_user_settings(user_id, validated_settings)
+            # Ensure runtime reads see updated per-user overrides immediately.
+            try:
+                from shelfmark.core.config import config as app_config
+                app_config.refresh(force=True)
+            except Exception:
+                pass
+
+        updated = user_db.get_user(user_id=user_id)
+        result = _serialize_user(
+            updated,
+            g.auth_mode,
+            security_config=security_config,
+        )
+        result["settings"] = user_db.get_user_settings(user_id)
+        logger.info(f"Admin updated user {user_id}")
+        return jsonify(result)
+
+    @app.route("/api/admin/users/sync-cwa", methods=["POST"])
+    @_require_admin
+    def admin_sync_cwa_users():
+        """Manually sync users from Calibre-Web into users.db."""
+        if g.auth_mode != AUTH_SOURCE_CWA:
+            return jsonify({
+                "error": "CWA sync is only available when CWA authentication is enabled",
+            }), 400
+
+        try:
+            summary = _sync_all_cwa_users(user_db)
+        except FileNotFoundError:
+            return jsonify({
+                "error": "Calibre-Web database is not available",
+                "message": "Verify app.db is mounted and readable at /auth/app.db.",
+            }), 503
+        except Exception as exc:
+            logger.error(f"Failed to sync CWA users: {exc}")
+            return jsonify({
+                "error": "Failed to sync users from Calibre-Web",
+            }), 500
+
+        message = (
+            f"Synced {summary['total']} CWA users "
+            f"({summary['created']} created, {summary['updated']} updated, "
+            f"{summary.get('deleted', 0)} deleted)."
+        )
+        logger.info(message)
+        return jsonify({
+            "success": True,
+            "message": message,
+            **summary,
+        })
+
+    register_admin_settings_routes(app, user_db, _require_admin)
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["DELETE"])
+    @_require_admin
+    def admin_delete_user(user_id):
+        """Delete a user."""
+        # Prevent self-deletion
+        if session.get("db_user_id") == user_id:
+            return jsonify({"error": "Cannot delete your own account"}), 400
+
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        auth_source = normalize_auth_source(
+            user.get("auth_source"),
+            user.get("oidc_subject"),
+        )
+        if auth_source == AUTH_SOURCE_CWA and auth_source == g.auth_mode:
+            return jsonify({
+                "error": f"Cannot delete active {auth_source.upper()} users",
+                "message": f"{auth_source.upper()} users are automatically re-provisioned on login.",
+            }), 400
+
+        # Allow deleting the last local admin account.
+        # Auth mode resolution automatically falls back to "none" when no
+        # local password admin remains.
+
+        user_db.delete_user(user_id)
+        logger.info(f"Admin deleted user {user_id}: {user['username']}")
+        return jsonify({"success": True})

shelfmark/core/admin_settings_routes.py

@@ -0,0 +1,258 @@
+"""Admin settings-introspection routes and settings validation helpers."""
+
+from typing import Any, Callable
+
+from flask import Flask, jsonify, request
+
+from shelfmark.config.notifications_settings import (
+    build_notification_test_result,
+    is_valid_notification_url,
+    normalize_notification_routes,
+)
+from shelfmark.config.users_settings import validate_search_preference_value
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_settings_overrides import (
+    build_user_preferences_payload as _build_user_preferences_payload,
+    get_ordered_user_overridable_fields as _get_ordered_user_overridable_fields,
+    get_settings_registry as _get_settings_registry,
+)
+from shelfmark.core.user_db import UserDB
+from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
+
+
+def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
+    settings_registry = _get_settings_registry()
+    field_map = settings_registry.get_settings_field_map()
+    overridable_map = settings_registry.get_user_overridable_fields()
+
+    valid: dict[str, Any] = {}
+    errors: list[str] = []
+    for key, value in settings.items():
+        if key not in field_map:
+            errors.append(f"Unknown setting: {key}")
+        elif key not in overridable_map:
+            errors.append(f"Setting not user-overridable: {key}")
+        else:
+            # null means "clear the per-user override; use global default"
+            if value is None:
+                valid[key] = None
+                continue
+
+            if key in {"REQUEST_POLICY_DEFAULT_EBOOK", "REQUEST_POLICY_DEFAULT_AUDIOBOOK"}:
+                if parse_policy_mode(value) is None:
+                    errors.append(f"Invalid policy mode for {key}: {value}")
+                    continue
+
+            if key == "REQUEST_POLICY_RULES":
+                normalized_rules, rule_errors = validate_policy_rules(value)
+                if rule_errors:
+                    errors.extend(rule_errors)
+                    continue
+                valid[key] = normalized_rules
+                continue
+
+            if key == "USER_NOTIFICATION_ROUTES":
+                normalized_routes = normalize_notification_routes(value)
+                invalid_count = sum(
+                    1
+                    for row in normalized_routes
+                    if row.get("url") and not is_valid_notification_url(str(row.get("url")))
+                )
+                if invalid_count:
+                    errors.append(
+                        (
+                            f"Invalid value for {key}: found {invalid_count} invalid URL(s). "
+                            "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+                        )
+                    )
+                    continue
+                valid[key] = normalized_routes
+                continue
+
+            normalized_search_value, search_validation_error = validate_search_preference_value(key, value)
+            if search_validation_error:
+                errors.append(search_validation_error)
+                continue
+            if key in {
+                "SEARCH_MODE",
+                "METADATA_PROVIDER",
+                "METADATA_PROVIDER_AUDIOBOOK",
+                "DEFAULT_RELEASE_SOURCE",
+            }:
+                valid[key] = normalized_search_value
+                continue
+
+            valid[key] = value
+
+    return valid, errors
+
+
+def build_user_notification_test_response(
+    *,
+    user_id: int,
+    payload: Any,
+) -> tuple[dict[str, Any], int]:
+    from shelfmark.core.config import config as app_config
+
+    routes_input = app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=user_id)
+    if isinstance(payload, dict):
+        if "USER_NOTIFICATION_ROUTES" in payload:
+            routes_input = payload.get("USER_NOTIFICATION_ROUTES")
+        elif "routes" in payload:
+            routes_input = payload.get("routes")
+
+    result = build_notification_test_result(routes_input, scope_label="personal")
+    status_code = 200 if result.get("success", False) else 400
+    return result, status_code
+
+
+def register_admin_settings_routes(
+    app: Flask,
+    user_db: UserDB,
+    require_admin: Callable[[Callable[..., Any]], Callable[..., Any]],
+) -> None:
+    @app.route("/api/admin/download-defaults", methods=["GET"])
+    @require_admin
+    def admin_download_defaults():
+        config = load_config_file("downloads")
+        defaults = {
+            key: ("" if (value := config.get(key, field.default)) is None else value)
+            for key, field in _get_ordered_user_overridable_fields("downloads")
+        }
+
+        security_config = load_config_file("security")
+        defaults["OIDC_ADMIN_GROUP"] = security_config.get("OIDC_ADMIN_GROUP", "")
+        defaults["OIDC_USE_ADMIN_GROUP"] = security_config.get("OIDC_USE_ADMIN_GROUP", True)
+        defaults["OIDC_AUTO_PROVISION"] = security_config.get("OIDC_AUTO_PROVISION", True)
+        return jsonify(defaults)
+
+    @app.route("/api/admin/booklore-options", methods=["GET"])
+    @require_admin
+    def admin_booklore_options():
+        from shelfmark.core import admin_routes
+
+        return jsonify({
+            "libraries": admin_routes.get_booklore_library_options(),
+            "paths": admin_routes.get_booklore_path_options(),
+        })
+
+    @app.route("/api/admin/users/<int:user_id>/delivery-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_delivery_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "downloads")
+        except ValueError:
+            return jsonify({"error": "Downloads settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/search-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_search_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "search_mode")
+        except ValueError:
+            return jsonify({"error": "Search mode settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/notification-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_notification_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "notifications")
+        except ValueError:
+            return jsonify({"error": "Notifications settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/notification-preferences/test", methods=["POST"])
+    @require_admin
+    def admin_test_notification_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        payload = request.get_json(silent=True)
+        result, status_code = build_user_notification_test_response(
+            user_id=user_id,
+            payload=payload,
+        )
+        return jsonify(result), status_code
+
+    @app.route("/api/admin/settings/overrides-summary", methods=["GET"])
+    @require_admin
+    def admin_settings_overrides_summary():
+        settings_registry = _get_settings_registry()
+
+        tab_name = (request.args.get("tab") or "downloads").strip()
+        if not settings_registry.get_settings_tab(tab_name):
+            return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404
+
+        overridable_keys = list(settings_registry.get_user_overridable_fields(tab_name=tab_name))
+        keys_payload: dict[str, dict[str, Any]] = {}
+
+        for user_record in user_db.list_users():
+            user_settings = user_db.get_user_settings(user_record["id"])
+            if not isinstance(user_settings, dict):
+                continue
+
+            for key in overridable_keys:
+                if key not in user_settings or user_settings[key] is None:
+                    continue
+                entry = keys_payload.setdefault(key, {"count": 0, "users": []})
+                entry["users"].append({
+                    "userId": user_record["id"],
+                    "username": user_record["username"],
+                    "value": user_settings[key],
+                })
+
+        for summary in keys_payload.values():
+            summary["count"] = len(summary["users"])
+
+        return jsonify({"tab": tab_name, "keys": keys_payload})
+
+    @app.route("/api/admin/users/<int:user_id>/effective-settings", methods=["GET"])
+    @require_admin
+    def admin_get_effective_settings(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        from shelfmark.core.config import config as app_config
+        from shelfmark.core.settings_registry import is_value_from_env
+
+        field_map = _get_settings_registry().get_user_overridable_fields()
+        user_settings = user_db.get_user_settings(user_id)
+        tab_config_cache: dict[str, dict[str, Any]] = {}
+        effective: dict[str, dict[str, Any]] = {}
+
+        for key, (field, tab_name) in sorted(field_map.items()):
+            value = app_config.get(key, field.default, user_id=user_id)
+            source = "default"
+
+            if field.env_supported and is_value_from_env(field):
+                source = "env_var"
+            elif key in user_settings and user_settings[key] is not None:
+                source = "user_override"
+                value = user_settings[key]
+            else:
+                tab_config = tab_config_cache.setdefault(tab_name, load_config_file(tab_name))
+                if key in tab_config:
+                    source = "global_config"
+
+            effective[key] = {"value": value, "source": source}
+
+        return jsonify(effective)

shelfmark/core/auth_modes.py

@@ -0,0 +1,150 @@
+"""Authentication mode, auth-source normalization, and admin access policy helpers."""
+
+import os
+from typing import Any, Mapping
+
+AUTH_SOURCE_BUILTIN = "builtin"
+AUTH_SOURCE_OIDC = "oidc"
+AUTH_SOURCE_PROXY = "proxy"
+AUTH_SOURCE_CWA = "cwa"
+AUTH_SOURCES = (
+    AUTH_SOURCE_BUILTIN,
+    AUTH_SOURCE_OIDC,
+    AUTH_SOURCE_PROXY,
+    AUTH_SOURCE_CWA,
+)
+AUTH_SOURCE_SET = frozenset(AUTH_SOURCES)
+_ALWAYS_ADMIN_SETTINGS_TABS = frozenset({"security", "users"})
+
+
+def has_local_password_admin(user_db: Any | None = None) -> bool:
+    """Return True when at least one local admin with a password exists."""
+    try:
+        db = user_db
+        if db is None:
+            from shelfmark.core.user_db import UserDB
+
+            config_root = os.environ.get("CONFIG_DIR", "/config")
+            db = UserDB(os.path.join(config_root, "users.db"))
+            db.initialize()
+
+        return db.has_admin_with_password()
+    except Exception:
+        return False
+
+
+def normalize_auth_source(
+    source: Any,
+    oidc_subject: Any = None,
+) -> str:
+    """Resolve a stable auth source value from persisted fields."""
+    normalized = str(source or "").strip().lower()
+    if normalized in AUTH_SOURCE_SET:
+        return normalized
+    if oidc_subject:
+        return AUTH_SOURCE_OIDC
+    return AUTH_SOURCE_BUILTIN
+
+
+def determine_auth_mode(
+    security_config: Mapping[str, Any],
+    cwa_db_path: Any | None,
+    *,
+    has_local_admin: bool = True,
+) -> str:
+    """Determine active auth mode from security config and runtime prerequisites."""
+    auth_mode = security_config.get("AUTH_METHOD", "none")
+
+    if auth_mode == AUTH_SOURCE_CWA and cwa_db_path:
+        return AUTH_SOURCE_CWA
+
+    if auth_mode == AUTH_SOURCE_BUILTIN and has_local_admin:
+        return AUTH_SOURCE_BUILTIN
+
+    if auth_mode == AUTH_SOURCE_PROXY and security_config.get("PROXY_AUTH_USER_HEADER"):
+        return AUTH_SOURCE_PROXY
+
+    if (
+        auth_mode == AUTH_SOURCE_OIDC
+        and has_local_admin
+        and security_config.get("OIDC_DISCOVERY_URL")
+        and security_config.get("OIDC_CLIENT_ID")
+    ):
+        return AUTH_SOURCE_OIDC
+
+    return "none"
+
+
+def load_active_auth_mode(
+    cwa_db_path: Any | None,
+    *,
+    user_db: Any | None = None,
+) -> str:
+    """Resolve active auth mode using current security config and runtime prerequisites."""
+    try:
+        from shelfmark.core.settings_registry import load_config_file
+
+        security_config = load_config_file("security")
+        return determine_auth_mode(
+            security_config,
+            cwa_db_path,
+            has_local_admin=has_local_password_admin(user_db),
+        )
+    except Exception:
+        return "none"
+
+
+def is_user_active_for_auth_mode(user: Mapping[str, Any], auth_mode: str) -> bool:
+    """Return whether a user can authenticate under the current auth mode."""
+    source = normalize_auth_source(user.get("auth_source"), user.get("oidc_subject"))
+    if source == AUTH_SOURCE_BUILTIN:
+        return auth_mode in (AUTH_SOURCE_BUILTIN, AUTH_SOURCE_OIDC)
+    return source == auth_mode
+
+
+def is_settings_or_onboarding_path(path: str) -> bool:
+    """Return True when request path targets protected admin settings routes."""
+    return path.startswith("/api/settings") or path.startswith("/api/onboarding")
+
+
+def get_settings_tab_from_path(path: str) -> str | None:
+    """Extract tab name from /api/settings/<tab>[...] paths."""
+    if not path.startswith("/api/settings/"):
+        return None
+
+    suffix = path[len("/api/settings/"):]
+    if not suffix:
+        return None
+
+    return suffix.split("/", 1)[0] or None
+
+
+def should_restrict_settings_to_admin(
+    _users_config: Mapping[str, Any],
+) -> bool:
+    """Settings/onboarding is always admin-only."""
+    return True
+
+
+def requires_admin_for_settings_access(
+    path: str,
+    users_config: Mapping[str, Any],
+) -> bool:
+    """Return whether this settings/onboarding request requires admin privileges."""
+    tab_name = get_settings_tab_from_path(path)
+    if tab_name in _ALWAYS_ADMIN_SETTINGS_TABS:
+        return True
+
+    return should_restrict_settings_to_admin(users_config)
+
+
+def get_auth_check_admin_status(
+    _auth_mode: str,
+    _users_config: Mapping[str, Any],
+    session_data: Mapping[str, Any],
+) -> bool:
+    """Resolve /api/auth/check `is_admin` as the session's real admin role."""
+    if "user_id" not in session_data:
+        return False
+
+    return bool(session_data.get("is_admin", False))

shelfmark/core/cache.py

@@ -0,0 +1,180 @@
+"""Thread-safe in-memory cache with TTL support."""
+
+import threading
+import time
+from dataclasses import dataclass
+from functools import wraps
+from typing import Any, Callable, Dict, Optional, TypeVar
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+T = TypeVar("T")
+
+
+@dataclass
+class CacheEntry:
+    """A cached value with expiration time."""
+    value: Any
+    expires_at: float
+
+
+class CacheService:
+    """Thread-safe in-memory cache with TTL support."""
+
+    def __init__(self, max_size: int = 1000):
+        """Initialize cache with max_size entries before eviction."""
+        self._cache: Dict[str, CacheEntry] = {}
+        self._lock = threading.Lock()
+        self._max_size = max_size
+
+    def get(self, key: str) -> Optional[Any]:
+        """Get cached value if not expired."""
+        with self._lock:
+            entry = self._cache.get(key)
+            if entry is None:
+                return None
+
+            if time.time() > entry.expires_at:
+                del self._cache[key]
+                return None
+
+            return entry.value
+
+    def set(self, key: str, value: Any, ttl: int) -> None:
+        """Cache value with TTL in seconds."""
+        with self._lock:
+            # Evict oldest entries if at capacity
+            if len(self._cache) >= self._max_size:
+                self._evict_oldest()
+
+            self._cache[key] = CacheEntry(
+                value=value,
+                expires_at=time.time() + ttl
+            )
+
+    def invalidate(self, key: str) -> bool:
+        """Remove specific cache entry. Returns True if found."""
+        with self._lock:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+
+    def invalidate_prefix(self, prefix: str) -> int:
+        """Remove all cache entries whose keys start with prefix."""
+        with self._lock:
+            matching_keys = [key for key in self._cache if key.startswith(prefix)]
+            for key in matching_keys:
+                del self._cache[key]
+            return len(matching_keys)
+
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        with self._lock:
+            self._cache.clear()
+
+    def cleanup_expired(self) -> int:
+        """Remove all expired entries. Returns count removed."""
+        with self._lock:
+            now = time.time()
+            expired_keys = [
+                key for key, entry in self._cache.items()
+                if entry.expires_at < now
+            ]
+            for key in expired_keys:
+                del self._cache[key]
+            return len(expired_keys)
+
+    def _evict_oldest(self) -> None:
+        """Evict ~10% of oldest entries. Called with lock held."""
+        if not self._cache:
+            return
+
+        # Remove ~10% of entries, oldest first
+        entries_to_remove = max(1, len(self._cache) // 10)
+        sorted_entries = sorted(
+            self._cache.items(),
+            key=lambda x: x[1].expires_at
+        )
+
+        for key, _ in sorted_entries[:entries_to_remove]:
+            del self._cache[key]
+
+    def stats(self) -> Dict[str, int]:
+        """Get cache statistics (size, max_size)."""
+        with self._lock:
+            return {
+                "size": len(self._cache),
+                "max_size": self._max_size
+            }
+
+
+# Global cache instance for metadata providers
+_metadata_cache = CacheService(max_size=1000)
+
+
+def get_metadata_cache() -> CacheService:
+    """Get the global metadata cache instance."""
+    return _metadata_cache
+
+
+def cache_key(*args, **kwargs) -> str:
+    """Generate cache key from arguments."""
+    parts = [str(arg) for arg in args]
+    parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
+    return ":".join(parts)
+
+
+def cacheable(
+    ttl: Optional[int] = None,
+    ttl_key: Optional[str] = None,
+    ttl_default: int = 300,
+    key_prefix: str = ""
+):
+    """Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            # Check if metadata caching is enabled
+            from shelfmark.core.config import config
+
+            if not config.get("METADATA_CACHE_ENABLED", True):
+                # Caching disabled, execute function directly
+                return func(*args, **kwargs)
+
+            # Determine TTL: static or from config
+            if ttl is not None:
+                effective_ttl = ttl
+            elif ttl_key:
+                effective_ttl = config.get(ttl_key, ttl_default)
+            else:
+                effective_ttl = ttl_default
+
+            # Generate cache key from function name and arguments
+            # Skip 'self' argument if present (first arg of method)
+            cache_args = args[1:] if args and hasattr(args[0], func.__name__) else args
+
+            key = cache_key(
+                key_prefix or func.__name__,
+                *cache_args,
+                **kwargs
+            )
+
+            # Check cache
+            cached = _metadata_cache.get(key)
+            if cached is not None:
+                return cached
+
+            # Execute function and cache result
+            result = func(*args, **kwargs)
+
+            # Only cache non-None results
+            if result is not None:
+                _metadata_cache.set(key, result, effective_ttl)
+
+            return result
+
+        return wrapper
+    return decorator

shelfmark/core/config.py

@@ -0,0 +1,277 @@
+"""Configuration singleton with ENV > config file > default resolution."""
+
+import os
+import sqlite3
+import time
+from threading import Lock
+from typing import Any, Dict, Optional
+
+# Import lazily to avoid circular imports
+_registry_module = None
+_env_module = None
+_user_db_module = None
+
+
+def _get_registry():
+    """Lazy import of settings registry to avoid circular imports."""
+    global _registry_module
+    if _registry_module is None:
+        from shelfmark.core import settings_registry
+        _registry_module = settings_registry
+    return _registry_module
+
+
+def _get_env():
+    """Lazy import of env module for fallback values."""
+    global _env_module
+    if _env_module is None:
+        from shelfmark.config import env
+        _env_module = env
+    return _env_module
+
+
+def _get_user_db_module():
+    """Lazy import of user DB module to avoid optional dependency loops."""
+    global _user_db_module
+    if _user_db_module is None:
+        from shelfmark.core.user_db import UserDB
+        _user_db_module = UserDB
+    return _user_db_module
+
+
+class Config:
+    """
+    Dynamic configuration singleton that provides live settings access.
+
+    Settings are resolved with priority: ENV var > config file > default.
+    Values are cached for performance and can be refreshed when settings change.
+    """
+
+    _instance: Optional['Config'] = None
+    _lock = Lock()
+    def __new__(cls) -> 'Config':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+        self._cache: Dict[str, Any] = {}
+        self._field_map: Dict[str, tuple] = {}  # key -> (field, tab_name)
+        self._cache_lock = Lock()
+        self._user_settings_cache: Dict[int, Dict[str, Any]] = {}
+        self._user_settings_cache_lock = Lock()
+        self._user_db = None
+        self._user_db_load_attempted = False
+        self._initialized = True
+        self._loaded = False
+        self._last_refresh_time: float = 0.0
+
+    def _ensure_loaded(self) -> None:
+        """Ensure settings are loaded from the registry."""
+        if self._loaded:
+            return
+        with self._cache_lock:
+            if self._loaded:
+                return
+            self._load_settings()
+
+    def _load_settings(self) -> None:
+        """Load all settings from the registry."""
+        # Ensure all settings modules are imported before loading
+        # This handles cases where config is accessed before settings are registered
+        try:
+            import shelfmark.config.settings  # noqa: F401 - main app settings
+            import shelfmark.config.notifications_settings  # noqa: F401 - notifications settings
+            import shelfmark.release_sources  # noqa: F401 - plugin settings
+            import shelfmark.metadata_providers  # noqa: F401 - plugin settings
+        except ImportError:
+            pass
+
+        registry = _get_registry()
+
+        # On first load, sync ENV values to config files
+        # This ensures ENV values persist even if ENV vars are later removed
+        if not hasattr(self, '_env_synced'):
+            registry.sync_env_to_config()
+            self._env_synced = True
+
+        # Build field map from all registered tabs
+        self._field_map.clear()
+        self._cache.clear()
+
+        for tab in registry.get_all_settings_tabs():
+            for field in tab.fields:
+                # Skip action buttons and headings - they don't have values
+                if isinstance(field, (registry.ActionButton, registry.HeadingField)):
+                    continue
+
+                key = field.key
+                self._field_map[key] = (field, tab.name)
+
+                # Load current value
+                value = registry.get_setting_value(field, tab.name)
+                self._cache[key] = value
+
+        self._loaded = True
+
+    def refresh(self, force: bool = False) -> None:
+        """
+        Refresh all cached settings from config files.
+
+        Call this after settings are updated via the UI to ensure
+        the config singleton reflects the new values.
+
+        Multiple calls within a short window (50 ms) are coalesced to
+        avoid redundant disk I/O when several helpers each call refresh()
+        during the same request.  Pass ``force=True`` to bypass the guard
+        (e.g. after a settings write).
+        """
+        now = time.monotonic()
+        if not force and (now - self._last_refresh_time) < 0.05:
+            return
+
+        with self._cache_lock:
+            self._loaded = False
+            self._load_settings()
+        with self._user_settings_cache_lock:
+            self._user_settings_cache.clear()
+        self._user_db = None
+        self._user_db_load_attempted = False
+        self._last_refresh_time = time.monotonic()
+
+    def _get_user_db(self):
+        """Get or initialize a UserDB handle if available."""
+        if self._user_db is not None:
+            return self._user_db
+        if self._user_db_load_attempted:
+            return None
+
+        self._user_db_load_attempted = True
+        try:
+            user_db_cls = _get_user_db_module()
+            db_path = os.path.join(os.environ.get("CONFIG_DIR", "/config"), "users.db")
+            user_db = user_db_cls(db_path)
+            user_db.initialize()
+            self._user_db = user_db
+            return self._user_db
+        except Exception:
+            # Multi-user support is optional; fall back to global config when unavailable.
+            return None
+
+    def _get_user_settings(self, user_id: int) -> Dict[str, Any]:
+        """Get cached per-user settings from user DB."""
+        with self._user_settings_cache_lock:
+            if user_id in self._user_settings_cache:
+                return self._user_settings_cache[user_id]
+
+        user_db = self._get_user_db()
+        if user_db is None:
+            return {}
+
+        try:
+            settings = user_db.get_user_settings(user_id)
+        except (sqlite3.OperationalError, OSError, ValueError, TypeError):
+            return {}
+
+        if not isinstance(settings, dict):
+            settings = {}
+
+        with self._user_settings_cache_lock:
+            self._user_settings_cache[user_id] = settings
+        return settings
+
+    def _get_user_override(self, user_id: int, key: str) -> Any:
+        """Get a user override for a specific key."""
+        user_settings = self._get_user_settings(user_id)
+        return user_settings.get(key)
+
+    def get(self, key: str, default: Any = None, user_id: Optional[int] = None) -> Any:
+        """
+        Get a setting value by key.
+
+        Args:
+            key: The setting key (e.g., 'MAX_RETRY')
+            default: Default value if setting not found
+            user_id: Optional DB user ID for per-user setting overrides
+
+        Returns:
+            The setting value, or default if not found
+        """
+        self._ensure_loaded()
+
+        if key in self._field_map:
+            field, _ = self._field_map[key]
+            registry = _get_registry()
+
+            # Deployment-level ENV values always win.
+            if field.env_supported and registry.is_value_from_env(field):
+                return self._cache.get(key, default)
+
+            # User overrides are only available for explicitly overridable fields.
+            if user_id is not None and getattr(field, "user_overridable", False):
+                user_value = self._get_user_override(user_id, key)
+                if user_value is not None:
+                    return user_value
+
+        return self._cache.get(key, default)
+
+    def __getattr__(self, name: str) -> Any:
+        """
+        Allow attribute-style access to settings.
+
+        Example: config.MAX_RETRY instead of config.get('MAX_RETRY')
+        """
+        # Avoid recursion for internal attributes
+        if name.startswith('_'):
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
+
+        self._ensure_loaded()
+
+        if name in self._cache:
+            return self._cache[name]
+
+        # Fallback to env module for settings not in registry
+        # This ensures backward compatibility during migration
+        env = _get_env()
+        if hasattr(env, name):
+            return getattr(env, name)
+
+        raise AttributeError(f"Setting '{name}' not found in config or env")
+
+    def is_from_env(self, key: str) -> bool:
+        """
+        Check if a setting's value comes from an environment variable.
+
+        Args:
+            key: The setting key
+
+        Returns:
+            True if the value is set via ENV var, False otherwise
+        """
+        self._ensure_loaded()
+
+        if key not in self._field_map:
+            return False
+
+        field, _ = self._field_map[key]
+        registry = _get_registry()
+        return registry.is_value_from_env(field)
+
+    def get_all(self) -> Dict[str, Any]:
+        """
+        Get all cached settings as a dictionary.
+
+        Returns:
+            Dict of all setting keys to their current values
+        """
+        self._ensure_loaded()
+        return dict(self._cache)
+
+
+# Global singleton instance
+config = Config()

shelfmark/core/cwa_user_sync.py

@@ -0,0 +1,94 @@
+"""Helpers for provisioning and syncing Calibre-Web users into users.db."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from shelfmark.core.auth_modes import AUTH_SOURCE_CWA, normalize_auth_source
+from shelfmark.core.external_user_linking import upsert_external_user
+from shelfmark.core.user_db import UserDB
+
+_CWA_ALIAS_SUFFIX = "__cwa"
+
+
+def _normalize_email(value: Any) -> str | None:
+    if value is None:
+        return None
+    email = str(value).strip()
+    return email or None
+
+
+def upsert_cwa_user(
+    user_db: UserDB,
+    cwa_username: str,
+    cwa_email: str | None,
+    role: str,
+    context: str | None = None,
+) -> tuple[dict[str, Any], str]:
+    """Create/update a CWA-backed user with collision-safe matching."""
+    normalized_email = _normalize_email(cwa_email)
+    collision_strategy = "alias" if normalized_email else "takeover"
+    user, action = upsert_external_user(
+        user_db,
+        auth_source="cwa",
+        username=cwa_username,
+        email=normalized_email,
+        role=role,
+        allow_email_link=True,
+        collision_strategy=collision_strategy,
+        alias_suffix=_CWA_ALIAS_SUFFIX,
+        context=context,
+    )
+    if user is None:
+        raise RuntimeError("Unexpected CWA user sync result: no user returned")
+    return user, action
+
+
+def sync_cwa_users_from_rows(
+    user_db: UserDB,
+    rows: Iterable[tuple[Any, Any, Any]],
+) -> dict[str, int]:
+    """Sync CWA users from raw `(name, role_flags, email)` rows."""
+    created = 0
+    updated = 0
+    active_cwa_user_ids: set[int] = set()
+    for username, role_flags, email in rows:
+        normalized_username = str(username or "").strip()
+        if not normalized_username:
+            continue
+
+        role = "admin" if (int(role_flags or 0) & 1) == 1 else "user"
+        user, action = upsert_cwa_user(
+            user_db,
+            cwa_username=normalized_username,
+            cwa_email=_normalize_email(email),
+            role=role,
+            context="cwa_manual_sync",
+        )
+        active_cwa_user_ids.add(int(user["id"]))
+        if action == "created":
+            created += 1
+        else:
+            updated += 1
+
+    deleted = 0
+    for existing_user in user_db.list_users():
+        if normalize_auth_source(
+            existing_user.get("auth_source"),
+            existing_user.get("oidc_subject"),
+        ) != AUTH_SOURCE_CWA:
+            continue
+
+        existing_id = int(existing_user.get("id") or 0)
+        if existing_id in active_cwa_user_ids:
+            continue
+
+        user_db.delete_user(existing_id)
+        deleted += 1
+
+    return {
+        "created": created,
+        "updated": updated,
+        "deleted": deleted,
+        "total": created + updated,
+    }

shelfmark/core/download_history_service.py

@@ -0,0 +1,306 @@
+"""Persistence helpers for canonical download activity rows."""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from typing import Any
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.models import TERMINAL_QUEUE_STATUSES
+from shelfmark.core.request_helpers import normalize_optional_positive_int, normalize_optional_text, now_utc_iso
+
+logger = setup_logger(__name__)
+
+
+VALID_TERMINAL_STATUSES = frozenset(s.value for s in TERMINAL_QUEUE_STATUSES)
+ACTIVE_DOWNLOAD_STATUS = "active"
+VALID_ORIGINS = frozenset({"direct", "requested"})
+
+
+def _normalize_task_id(task_id: Any) -> str:
+    normalized = normalize_optional_text(task_id)
+    if normalized is None:
+        raise ValueError("task_id must be a non-empty string")
+    return normalized
+
+
+def _normalize_origin(origin: Any) -> str:
+    normalized = normalize_optional_text(origin)
+    if normalized is None:
+        return "direct"
+    lowered = normalized.lower()
+    if lowered not in VALID_ORIGINS:
+        raise ValueError("origin must be one of: direct, requested")
+    return lowered
+
+
+def _normalize_final_status(final_status: Any) -> str:
+    normalized = normalize_optional_text(final_status)
+    if normalized is None:
+        raise ValueError("final_status must be a non-empty string")
+    lowered = normalized.lower()
+    if lowered not in VALID_TERMINAL_STATUSES:
+        raise ValueError("final_status must be one of: complete, error, cancelled")
+    return lowered
+
+
+def _normalize_limit(value: Any, *, default: int, minimum: int, maximum: int) -> int:
+    if value is None:
+        return default
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("limit must be an integer") from exc
+    if parsed < minimum:
+        return minimum
+    if parsed > maximum:
+        return maximum
+    return parsed
+
+
+class DownloadHistoryService:
+    """Service for persisted canonical download activity rows."""
+
+    def __init__(self, db_path: str):
+        self._db_path = db_path
+        self._lock = threading.Lock()
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA foreign_keys = ON")
+        return conn
+
+    @staticmethod
+    def _row_to_dict(row: sqlite3.Row | None) -> dict[str, Any] | None:
+        return dict(row) if row is not None else None
+
+    @staticmethod
+    def _to_item_key(task_id: str) -> str:
+        return f"download:{task_id}"
+
+    @staticmethod
+    def _resolve_existing_download_path(value: Any) -> str | None:
+        normalized = normalize_optional_text(value)
+        if normalized is None:
+            return None
+        return normalized if os.path.exists(normalized) else None
+
+    @staticmethod
+    def to_download_payload(row: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "id": row.get("task_id"),
+            "title": row.get("title"),
+            "author": row.get("author"),
+            "format": row.get("format"),
+            "size": row.get("size"),
+            "preview": row.get("preview"),
+            "content_type": row.get("content_type"),
+            "source": row.get("source"),
+            "source_display_name": row.get("source_display_name"),
+            "status_message": row.get("status_message"),
+            "download_path": DownloadHistoryService._resolve_existing_download_path(row.get("download_path")),
+            "added_time": DownloadHistoryService._iso_to_epoch(row.get("queued_at")),
+            "user_id": row.get("user_id"),
+            "username": row.get("username"),
+            "request_id": row.get("request_id"),
+        }
+
+    @staticmethod
+    def _iso_to_epoch(value: Any) -> float | None:
+        if not isinstance(value, str) or not value.strip():
+            return None
+        normalized = value.strip().replace("Z", "+00:00")
+        try:
+            parsed = datetime.fromisoformat(normalized)
+        except ValueError:
+            return None
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=timezone.utc)
+        return parsed.timestamp()
+
+    @classmethod
+    def to_history_row(cls, row: dict[str, Any], *, dismissed_at: str) -> dict[str, Any]:
+        task_id = str(row.get("task_id") or "").strip()
+        item_key = cls._to_item_key(task_id)
+        download_payload = cls.to_download_payload(row)
+        # Clear stale progress messages for non-error terminal states.
+        if row.get("final_status") in ("complete", "cancelled"):
+            download_payload["status_message"] = None
+        return {
+            "id": item_key,
+            "user_id": row.get("user_id"),
+            "item_type": "download",
+            "item_key": item_key,
+            "dismissed_at": dismissed_at,
+            "snapshot": {
+                "kind": "download",
+                "download": download_payload,
+            },
+            "origin": row.get("origin"),
+            "final_status": row.get("final_status"),
+            "terminal_at": row.get("terminal_at"),
+            "request_id": row.get("request_id"),
+            "source_id": task_id or None,
+        }
+
+    def record_download(
+        self,
+        *,
+        task_id: str,
+        user_id: int | None,
+        username: str | None,
+        request_id: int | None,
+        source: str,
+        source_display_name: str | None,
+        title: str,
+        author: str | None,
+        format: str | None,
+        size: str | None,
+        preview: str | None,
+        content_type: str | None,
+        origin: str,
+    ) -> None:
+        """Record a download at queue time with final_status='active'.
+
+        On first queue: inserts a new row.
+        On retry (row already exists): resets the row back to 'active'
+        so the normal finalize path works when the retry completes.
+        """
+        normalized_task_id = _normalize_task_id(task_id)
+        normalized_user_id = normalize_optional_positive_int(user_id, "user_id")
+        normalized_request_id = normalize_optional_positive_int(request_id, "request_id")
+        normalized_source = normalize_optional_text(source)
+        if normalized_source is None:
+            raise ValueError("source must be a non-empty string")
+        normalized_title = normalize_optional_text(title)
+        if normalized_title is None:
+            raise ValueError("title must be a non-empty string")
+        normalized_origin = _normalize_origin(origin)
+        recorded_at = now_utc_iso()
+
+        with self._lock:
+            conn = self._connect()
+            try:
+                conn.execute(
+                    """
+                INSERT INTO download_history (
+                    task_id, user_id, username, request_id,
+                    source, source_display_name,
+                    title, author, format, size, preview, content_type,
+                    origin, final_status,
+                    status_message, download_path,
+                    queued_at, terminal_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', NULL, NULL, ?, ?)
+                ON CONFLICT(task_id) DO UPDATE SET
+                    final_status = 'active',
+                    status_message = NULL,
+                    download_path = NULL,
+                    terminal_at = ?
+                """,
+                    (
+                        normalized_task_id,
+                        normalized_user_id,
+                        normalize_optional_text(username),
+                        normalized_request_id,
+                        normalized_source,
+                        normalize_optional_text(source_display_name),
+                        normalized_title,
+                        normalize_optional_text(author),
+                        normalize_optional_text(format),
+                        normalize_optional_text(size),
+                        normalize_optional_text(preview),
+                        normalize_optional_text(content_type),
+                        normalized_origin,
+                        recorded_at,
+                        recorded_at,
+                        recorded_at,
+                    ),
+                )
+                conn.commit()
+            finally:
+                conn.close()
+
+    def finalize_download(
+        self,
+        *,
+        task_id: str,
+        final_status: str,
+        status_message: str | None = None,
+        download_path: str | None = None,
+    ) -> None:
+        """Update an existing download row to its terminal state."""
+        normalized_task_id = _normalize_task_id(task_id)
+        normalized_final_status = _normalize_final_status(final_status)
+        normalized_status_message = normalize_optional_text(status_message)
+        normalized_download_path = normalize_optional_text(download_path)
+        effective_terminal_at = now_utc_iso()
+
+        with self._lock:
+            conn = self._connect()
+            try:
+                cursor = conn.execute(
+                    """
+                    UPDATE download_history
+                    SET final_status = ?,
+                        status_message = ?,
+                        download_path = ?,
+                        terminal_at = ?
+                    WHERE task_id = ? AND final_status = 'active'
+                    """,
+                    (
+                        normalized_final_status,
+                        normalized_status_message,
+                        normalized_download_path,
+                        effective_terminal_at,
+                        normalized_task_id,
+                    ),
+                )
+                rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
+                if rowcount < 1:
+                    logger.warning(
+                        "finalize_download: no active row found for task_id=%s (may have been missed at queue time)",
+                        normalized_task_id,
+                    )
+                conn.commit()
+            finally:
+                conn.close()
+
+    def get_by_task_id(self, task_id: str) -> dict[str, Any] | None:
+        normalized_task_id = _normalize_task_id(task_id)
+        conn = self._connect()
+        try:
+            row = conn.execute(
+                "SELECT * FROM download_history WHERE task_id = ?",
+                (normalized_task_id,),
+            ).fetchone()
+            return self._row_to_dict(row)
+        finally:
+            conn.close()
+
+    def list_recent(
+        self,
+        *,
+        user_id: int | None,
+        limit: int = 200,
+    ) -> list[dict[str, Any]]:
+        normalized_user_id = normalize_optional_positive_int(user_id, "user_id")
+        normalized_limit = _normalize_limit(limit, default=200, minimum=1, maximum=1000)
+        query = "SELECT * FROM download_history"
+        params: list[Any] = []
+        if normalized_user_id is not None:
+            query += " WHERE user_id = ?"
+            params.append(normalized_user_id)
+        query += " ORDER BY terminal_at DESC, id DESC LIMIT ?"
+        params.append(normalized_limit)
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(query, params).fetchall()
+            return [dict(row) for row in rows]
+        finally:
+            conn.close()

shelfmark/core/external_user_linking.py

@@ -0,0 +1,283 @@
+"""Shared external identity matching and provisioning helpers."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Literal
+
+from shelfmark.core.auth_modes import normalize_auth_source
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.user_db import UserDB
+
+UNSET = object()
+
+CollisionStrategy = Literal["takeover", "suffix", "alias"]
+MatchReason = Literal[
+    "subject_match",
+    "existing_source_username_match",
+    "unique_email_match",
+]
+
+logger = setup_logger(__name__)
+
+
+def _normalize_username(value: Any) -> str:
+    return str(value or "").strip()
+
+
+def _normalize_email(value: Any) -> str | None:
+    if value is None:
+        return None
+    email = str(value).strip()
+    return email or None
+
+
+def _normalize_display_name(value: Any) -> str | None:
+    if value is None:
+        return None
+    name = str(value).strip()
+    return name or None
+
+
+def _email_key(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def _normalize_role(value: Any) -> str:
+    return "admin" if str(value or "").strip().lower() == "admin" else "user"
+
+
+def _get_by_subject(user_db: UserDB, subject_field: str | None, subject: str | None) -> dict[str, Any] | None:
+    if not subject_field or not subject:
+        return None
+    if subject_field == "oidc_subject":
+        return user_db.get_user(oidc_subject=subject)
+    return None
+
+
+def find_unique_user_by_email(user_db: UserDB, email: str | None) -> dict[str, Any] | None:
+    key = _email_key(_normalize_email(email))
+    if not key:
+        return None
+
+    matches = [u for u in user_db.list_users() if _email_key(u.get("email")) == key]
+    return matches[0] if len(matches) == 1 else None
+
+
+def find_external_user_match(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    username: str,
+    email: str | None,
+    subject_field: str | None = None,
+    subject: str | None = None,
+    allow_email_link: bool = False,
+) -> tuple[dict[str, Any] | None, MatchReason | None]:
+    """Find an existing local user that should be linked to an external identity."""
+    normalized_username = _normalize_username(username)
+    normalized_email = _normalize_email(email)
+
+    by_subject = _get_by_subject(user_db, subject_field, subject)
+    if by_subject is not None:
+        return by_subject, "subject_match"
+
+    by_username = user_db.get_user(username=normalized_username)
+    if by_username and normalize_auth_source(
+        by_username.get("auth_source"),
+        by_username.get("oidc_subject"),
+    ) == auth_source:
+        return by_username, "existing_source_username_match"
+
+    if allow_email_link:
+        return find_unique_user_by_email(user_db, normalized_email), "unique_email_match"
+    return None, None
+
+
+def _build_updates(
+    *,
+    auth_source: str,
+    role: str,
+    sync_role: bool,
+    email: str | None | object,
+    display_name: str | None | object,
+    subject_field: str | None,
+    subject: str | None,
+) -> dict[str, Any]:
+    updates: dict[str, Any] = {"auth_source": auth_source}
+    if sync_role:
+        updates["role"] = _normalize_role(role)
+    if email is not UNSET:
+        updates["email"] = _normalize_email(email)
+    if display_name is not UNSET:
+        updates["display_name"] = _normalize_display_name(display_name)
+    if subject_field == "oidc_subject" and subject:
+        updates["oidc_subject"] = subject
+    return updates
+
+
+def _next_suffix_username(user_db: UserDB, base_username: str) -> str:
+    candidate = base_username
+    suffix = 1
+    while user_db.get_user(username=candidate):
+        candidate = f"{base_username}_{suffix}"
+        suffix += 1
+    return candidate
+
+
+def _find_existing_alias_user(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    alias_base: str,
+) -> dict[str, Any] | None:
+    pattern = re.compile(rf"^{re.escape(alias_base)}(?:_\d+)?$")
+    candidates = [
+        user for user in user_db.list_users()
+        if pattern.match(str(user.get("username") or ""))
+        and normalize_auth_source(user.get("auth_source"), user.get("oidc_subject")) == auth_source
+    ]
+    if not candidates:
+        return None
+    return sorted(candidates, key=lambda user: int(user.get("id") or 0))[0]
+
+
+def _resolve_create_username(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    requested_username: str,
+    strategy: CollisionStrategy,
+    alias_suffix: str,
+) -> tuple[str | None, dict[str, Any] | None, str]:
+    existing = user_db.get_user(username=requested_username)
+    if not existing:
+        return requested_username, None, "new_username_available"
+
+    if strategy == "takeover":
+        return None, existing, "username_collision_takeover"
+
+    if strategy == "suffix":
+        return _next_suffix_username(user_db, requested_username), None, "username_collision_suffix"
+
+    alias_base = f"{requested_username}{alias_suffix}"
+    alias_existing = _find_existing_alias_user(
+        user_db,
+        auth_source=auth_source,
+        alias_base=alias_base,
+    )
+    if alias_existing is not None:
+        return None, alias_existing, "reuse_existing_alias"
+    return _next_suffix_username(user_db, alias_base), None, "username_collision_alias"
+
+
+def upsert_external_user(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    username: str,
+    role: str,
+    email: str | None | object = UNSET,
+    display_name: str | None | object = UNSET,
+    subject_field: str | None = None,
+    subject: str | None = None,
+    allow_email_link: bool = False,
+    sync_role: bool = True,
+    allow_create: bool = True,
+    collision_strategy: CollisionStrategy = "takeover",
+    alias_suffix: str | None = None,
+    context: str | None = None,
+) -> tuple[dict[str, Any] | None, str]:
+    """Create/update a user from an external auth identity.
+
+    Returns `(user, action)` where action is one of:
+    - `"updated"`
+    - `"created"`
+    - `"not_found"` (when `allow_create=False` and no link target exists)
+    """
+    normalized_username = _normalize_username(username)
+    if not normalized_username:
+        raise ValueError("External username is required")
+
+    normalized_email = _normalize_email(email) if email is not UNSET else None
+    normalized_display_name = (
+        _normalize_display_name(display_name) if display_name is not UNSET else None
+    )
+    normalized_role = _normalize_role(role)
+
+    matched, match_reason = find_external_user_match(
+        user_db,
+        auth_source=auth_source,
+        username=normalized_username,
+        email=normalized_email,
+        subject_field=subject_field,
+        subject=subject,
+        allow_email_link=allow_email_link,
+    )
+    updates = _build_updates(
+        auth_source=auth_source,
+        role=normalized_role,
+        sync_role=sync_role,
+        email=normalized_email if email is not UNSET else UNSET,
+        display_name=normalized_display_name if display_name is not UNSET else UNSET,
+        subject_field=subject_field,
+        subject=subject,
+    )
+    if matched is not None:
+        user_db.update_user(matched["id"], **updates)
+        mapped = user_db.get_user(user_id=matched["id"]) or matched
+        logger.info(
+            "External user mapped to existing Shelfmark user "
+            f"(source={auth_source}, context={context or 'unspecified'}, reason={match_reason}, "
+            f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
+            f"shelfmark_username={mapped['username']})"
+        )
+        return mapped, "updated"
+
+    if not allow_create:
+        logger.info(
+            "External user could not be mapped and creation is disabled "
+            f"(source={auth_source}, context={context or 'unspecified'}, "
+            f"external_username={normalized_username})"
+        )
+        return None, "not_found"
+
+    resolved_alias_suffix = alias_suffix or f"__{auth_source}"
+    create_username, takeover_target, create_reason = _resolve_create_username(
+        user_db,
+        auth_source=auth_source,
+        requested_username=normalized_username,
+        strategy=collision_strategy,
+        alias_suffix=resolved_alias_suffix,
+    )
+    if takeover_target is not None:
+        user_db.update_user(takeover_target["id"], **updates)
+        mapped = user_db.get_user(user_id=takeover_target["id"]) or takeover_target
+        logger.info(
+            "External user mapped to existing Shelfmark user "
+            f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
+            f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
+            f"shelfmark_username={mapped['username']})"
+        )
+        return mapped, "updated"
+
+    create_kwargs: dict[str, Any] = {
+        "username": create_username,
+        "auth_source": auth_source,
+        "role": normalized_role,
+    }
+    if email is not UNSET:
+        create_kwargs["email"] = normalized_email
+    if display_name is not UNSET:
+        create_kwargs["display_name"] = normalized_display_name
+    if subject_field == "oidc_subject" and subject:
+        create_kwargs["oidc_subject"] = subject
+
+    created = user_db.create_user(**create_kwargs)
+    logger.info(
+        "External user created Shelfmark user "
+        f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
+        f"external_username={normalized_username}, shelfmark_user_id={created['id']}, "
+        f"shelfmark_username={created['username']})"
+    )
+    return created, "created"

shelfmark/core/image_cache.py

@@ -0,0 +1,571 @@
+"""Disk-based image cache with LRU eviction."""
+
+import json
+import os
+import threading
+import time
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.network import get_ssl_verify
+
+logger = setup_logger(__name__)
+
+# Image type detection via magic bytes
+IMAGE_SIGNATURES = {
+    b'\xff\xd8\xff': ('image/jpeg', 'jpg'),
+    b'\x89PNG\r\n\x1a\n': ('image/png', 'png'),
+    b'GIF87a': ('image/gif', 'gif'),
+    b'GIF89a': ('image/gif', 'gif'),
+    b'RIFF': ('image/webp', 'webp'),  # WebP starts with RIFF
+}
+
+# HTTP headers for image fetching
+FETCH_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36',
+    'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+}
+
+# Maximum image size to fetch (5 MB)
+MAX_IMAGE_SIZE = 5 * 1024 * 1024
+
+# Negative cache TTL (for failed fetches) - 1 hour
+NEGATIVE_CACHE_TTL = 3600
+
+# Transient failure cache TTL (for timeouts/connection errors) - 60 seconds
+# Short enough to retry soon, long enough to prevent spam during one page view
+TRANSIENT_CACHE_TTL = 60
+
+
+def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
+    """Detect image type from magic bytes.
+
+    Args:
+        data: Image data bytes
+
+    Returns:
+        Tuple of (content_type, extension) or None if not recognized
+    """
+    for signature, (content_type, ext) in IMAGE_SIGNATURES.items():
+        if data.startswith(signature):
+            return content_type, ext
+
+    # Special case for WebP - check for WEBP after RIFF
+    if data.startswith(b'RIFF') and len(data) > 12 and data[8:12] == b'WEBP':
+        return 'image/webp', 'webp'
+
+    return None
+
+
+class ImageCacheService:
+    """Persistent image cache with LRU eviction and TTL support."""
+
+    def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0):
+        """Initialize the image cache.
+
+        Args:
+            cache_dir: Directory to store cached images
+            max_size_mb: Maximum cache size in megabytes
+            ttl_seconds: Time-to-live in seconds (0 = forever)
+        """
+        self.cache_dir = cache_dir
+        self.max_size_bytes = max_size_mb * 1024 * 1024
+        self.ttl_seconds = ttl_seconds
+        self.index_path = cache_dir / "cache_index.json"
+        self._lock = threading.RLock()
+        self._index: Dict[str, Dict[str, Any]] = {}
+
+        # Stats tracking
+        self._hits = 0
+        self._misses = 0
+
+        # Ensure cache directory exists
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load existing index and sync with files on disk (once at startup)
+        self._load_index()
+        self._sync_index_with_files()
+
+    def _load_index(self) -> None:
+        """Load cache index from disk."""
+        if not self.index_path.exists():
+            self._index = {}
+            return
+
+        try:
+            with open(self.index_path, 'r') as f:
+                self._index = json.load(f)
+        except (json.JSONDecodeError, IOError):
+            self._index = {}
+
+    def _sync_index_with_files(self) -> None:
+        """Sync cache index with actual files on disk.
+
+        - Adds entries for files that exist but aren't in index
+        - Removes entries for files that no longer exist (non-negative only)
+        - Preserves negative cache entries (they have no files)
+        """
+        image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
+        added_count = 0
+        removed_count = 0
+
+        # Build set of files that exist on disk
+        existing_files: Dict[str, Path] = {}
+        for file_path in self.cache_dir.iterdir():
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() not in image_extensions:
+                continue
+            existing_files[file_path.stem] = file_path
+
+        # Add files that aren't in the index
+        for cache_id, file_path in existing_files.items():
+            if cache_id in self._index:
+                continue
+
+            ext = file_path.suffix.lstrip('.')
+            stat = file_path.stat()
+
+            # Detect content type
+            try:
+                with open(file_path, 'rb') as f:
+                    header = f.read(16)
+                detected = _detect_image_type(header)
+                content_type = detected[0] if detected else f'image/{ext}'
+            except IOError:
+                content_type = f'image/{ext}'
+
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': stat.st_size,
+                'cached_at': stat.st_mtime,
+                'accessed_at': stat.st_mtime,
+            }
+            added_count += 1
+
+        # Remove index entries for missing files (skip negative cache entries)
+        stale_entries = []
+        for cache_id, entry in self._index.items():
+            if entry.get('negative', False):
+                continue  # Negative entries don't have files
+            if cache_id not in existing_files:
+                stale_entries.append(cache_id)
+
+        for cache_id in stale_entries:
+            del self._index[cache_id]
+            removed_count += 1
+
+        if added_count > 0 or removed_count > 0:
+            self._save_index()
+
+    def _save_index(self) -> None:
+        """Save cache index to disk."""
+        try:
+            # Write to temp file first, then rename for atomicity
+            temp_path = self.index_path.with_suffix('.tmp')
+            with open(temp_path, 'w') as f:
+                json.dump(self._index, f)
+            temp_path.rename(self.index_path)
+        except IOError:
+            pass
+
+    def _get_image_path(self, cache_id: str, ext: str) -> Path:
+        """Get the file path for a cached image."""
+        return self.cache_dir / f"{cache_id}.{ext}"
+
+    def _is_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a cache entry is expired."""
+        if self.ttl_seconds == 0:
+            return False
+        return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
+
+    def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
+        """Check if a negative cache entry is expired.
+
+        Transient failures (timeouts) expire after TRANSIENT_CACHE_TTL (60s).
+        Permanent failures (404s) expire after NEGATIVE_CACHE_TTL (1 hour).
+        """
+        if not entry.get('negative', False):
+            return False
+
+        cached_at = entry.get('cached_at', 0)
+        ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
+        return (time.time() - cached_at) > ttl
+
+    def _calculate_total_size(self) -> int:
+        """Calculate total size of cached images."""
+        return sum(entry.get('size', 0) for entry in self._index.values())
+
+    def _evict_if_needed(self, required_space: int = 0) -> None:
+        """Evict old entries if cache is over size limit.
+
+        Uses LRU eviction based on accessed_at timestamp.
+        """
+        current_size = self._calculate_total_size()
+        target_size = self.max_size_bytes - required_space
+
+        if current_size <= target_size:
+            return
+
+        # Sort entries by accessed_at (oldest first)
+        sorted_entries = sorted(
+            self._index.items(),
+            key=lambda x: x[1].get('accessed_at', 0)
+        )
+
+        evicted_count = 0
+        for cache_id, entry in sorted_entries:
+            if current_size <= target_size:
+                break
+
+            # Delete the image file
+            ext = entry.get('ext', 'jpg')
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                if image_path.exists():
+                    image_path.unlink()
+            except IOError:
+                pass
+
+            # Update tracking
+            current_size -= entry.get('size', 0)
+            del self._index[cache_id]
+            evicted_count += 1
+
+        if evicted_count > 0:
+            self._save_index()
+
+    def get(self, cache_id: str) -> Optional[Tuple[bytes, str]]:
+        """Get a cached image.
+
+        Args:
+            cache_id: Cache key (book ID or composite key)
+
+        Returns:
+            Tuple of (image_data, content_type) or None if not cached/expired
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+
+            # Try reloading from disk if not found (handles multiprocess case)
+            if not entry:
+                self._load_index()
+                entry = self._index.get(cache_id)
+                if not entry:
+                    self._misses += 1
+                    return None
+
+            # Check for negative cache (failed fetch)
+            if entry.get('negative', False):
+                if self._is_negative_expired(entry):
+                    # Negative cache expired, allow retry
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+                # Still in negative cache, return None (don't retry)
+                return None
+
+            # Check for expired entry
+            if self._is_expired(entry):
+                # Remove expired entry
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+                del self._index[cache_id]
+                self._save_index()
+                self._misses += 1
+                return None
+
+            # Try to read the cached image
+            ext = entry.get('ext', 'jpg')
+            content_type = entry.get('content_type', 'image/jpeg')
+            image_path = self._get_image_path(cache_id, ext)
+
+            try:
+                if not image_path.exists():
+                    # File missing, remove from index
+                    del self._index[cache_id]
+                    self._save_index()
+                    self._misses += 1
+                    return None
+
+                with open(image_path, 'rb') as f:
+                    data = f.read()
+
+                # Update accessed time
+                entry['accessed_at'] = time.time()
+                self._save_index()
+
+                self._hits += 1
+                return data, content_type
+
+            except IOError:
+                self._misses += 1
+                return None
+
+    def put(self, cache_id: str, data: bytes, content_type: str) -> bool:
+        """Store an image in the cache.
+
+        Args:
+            cache_id: Cache key
+            data: Image data bytes
+            content_type: MIME type of the image
+
+        Returns:
+            True if stored successfully
+        """
+        with self._lock:
+            # Detect image type for extension
+            detected = _detect_image_type(data)
+            if detected:
+                content_type, ext = detected
+            else:
+                # Fall back to content-type header
+                if 'jpeg' in content_type or 'jpg' in content_type:
+                    ext = 'jpg'
+                elif 'png' in content_type:
+                    ext = 'png'
+                elif 'gif' in content_type:
+                    ext = 'gif'
+                elif 'webp' in content_type:
+                    ext = 'webp'
+                else:
+                    ext = 'jpg'  # Default
+
+            image_size = len(data)
+
+            # Evict if needed to make room
+            self._evict_if_needed(image_size)
+
+            # Write image to disk
+            image_path = self._get_image_path(cache_id, ext)
+            try:
+                with open(image_path, 'wb') as f:
+                    f.write(data)
+            except IOError:
+                return False
+
+            # Update index
+            now = time.time()
+            self._index[cache_id] = {
+                'ext': ext,
+                'content_type': content_type,
+                'size': image_size,
+                'cached_at': now,
+                'accessed_at': now,
+                'negative': False,
+            }
+            self._save_index()
+            return True
+
+    def put_negative(self, cache_id: str, transient: bool = False) -> None:
+        """Store a negative cache entry (failed fetch).
+
+        Args:
+            cache_id: Cache key
+            transient: If True, uses shorter TTL (for timeouts/connection errors)
+        """
+        with self._lock:
+            self._index[cache_id] = {
+                'negative': True,
+                'transient': transient,
+                'cached_at': time.time(),
+            }
+            self._save_index()
+
+    def delete(self, cache_id: str) -> bool:
+        """Delete a single cache entry.
+
+        Args:
+            cache_id: Cache key
+
+        Returns:
+            True if entry existed and was deleted
+        """
+        with self._lock:
+            entry = self._index.get(cache_id)
+            if not entry:
+                return False
+
+            # Delete file if it exists
+            if not entry.get('negative', False):
+                ext = entry.get('ext', 'jpg')
+                image_path = self._get_image_path(cache_id, ext)
+                try:
+                    if image_path.exists():
+                        image_path.unlink()
+                except IOError:
+                    pass
+
+            del self._index[cache_id]
+            self._save_index()
+            return True
+
+    def clear(self) -> int:
+        """Clear all cached images.
+
+        Returns:
+            Number of entries cleared
+        """
+        with self._lock:
+            count = len(self._index)
+
+            # Delete all image files
+            for cache_id, entry in self._index.items():
+                if not entry.get('negative', False):
+                    ext = entry.get('ext', 'jpg')
+                    image_path = self._get_image_path(cache_id, ext)
+                    try:
+                        if image_path.exists():
+                            image_path.unlink()
+                    except IOError:
+                        pass
+
+            # Clear index
+            self._index = {}
+            self._save_index()
+
+            # Reset stats
+            self._hits = 0
+            self._misses = 0
+
+            return count
+
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dict with size, count, hit rate, etc.
+        """
+        with self._lock:
+            total_size = self._calculate_total_size()
+            entry_count = len(self._index)
+            negative_count = sum(1 for e in self._index.values() if e.get('negative', False))
+            total_requests = self._hits + self._misses
+            hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0
+
+            return {
+                'entry_count': entry_count,
+                'negative_count': negative_count,
+                'total_size_bytes': total_size,
+                'total_size_mb': round(total_size / (1024 * 1024), 2),
+                'max_size_mb': self.max_size_bytes / (1024 * 1024),
+                'hits': self._hits,
+                'misses': self._misses,
+                'hit_rate': round(hit_rate, 1),
+            }
+
+    def fetch_and_cache(self, cache_id: str, url: str) -> Optional[Tuple[bytes, str]]:
+        """Fetch an image from URL and cache it.
+
+        Args:
+            cache_id: Cache key
+            url: URL to fetch from
+
+        Returns:
+            Tuple of (image_data, content_type) or None on failure
+        """
+        try:
+
+            response = requests.get(
+                url,
+                timeout=(5, 10),
+                headers=FETCH_HEADERS,
+                stream=True,
+                verify=get_ssl_verify(url),
+            )
+            response.raise_for_status()
+
+            # Validate content type
+            content_type = response.headers.get('content-type', '')
+            if not content_type.startswith('image/'):
+                self.put_negative(cache_id)
+                return None
+
+            # Read with size limit
+            data = BytesIO()
+            for chunk in response.iter_content(chunk_size=8192):
+                data.write(chunk)
+                if data.tell() > MAX_IMAGE_SIZE:
+                    self.put_negative(cache_id)
+                    return None
+
+            image_data = data.getvalue()
+
+            if not image_data:
+                self.put_negative(cache_id)
+                return None
+
+            # Store in cache
+            if self.put(cache_id, image_data, content_type):
+                # Get the actual content type from detection
+                detected = _detect_image_type(image_data)
+                if detected:
+                    content_type = detected[0]
+                return image_data, content_type
+
+            return None
+
+        except requests.exceptions.Timeout:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.ConnectionError:
+            self.put_negative(cache_id, transient=True)
+            return None
+        except requests.exceptions.HTTPError as e:
+            is_404 = e.response is not None and e.response.status_code == 404
+            self.put_negative(cache_id, transient=not is_404)
+            return None
+        except Exception:
+            return None
+
+
+# Singleton instance (initialized lazily when config is available)
+_instance: Optional[ImageCacheService] = None
+_instance_lock = threading.Lock()
+
+
+def get_image_cache() -> ImageCacheService:
+    """Get the singleton image cache instance.
+
+    Lazily initializes using config values.
+    """
+    global _instance
+
+    if _instance is None:
+        with _instance_lock:
+            if _instance is None:
+                from shelfmark.core.config import config
+                from shelfmark.config.env import CONFIG_DIR
+
+                cache_dir = CONFIG_DIR / "covers"
+                max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
+                ttl_days = config.get("COVERS_CACHE_TTL", 0)
+                ttl_seconds = ttl_days * 86400 if ttl_days > 0 else 0
+
+                _instance = ImageCacheService(
+                    cache_dir=cache_dir,
+                    max_size_mb=max_size_mb,
+                    ttl_seconds=ttl_seconds,
+                )
+                logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
+
+    return _instance
+
+
+def reset_image_cache() -> None:
+    """Reset the singleton instance (for testing or config changes)."""
+    global _instance
+    with _instance_lock:
+        _instance = None

shelfmark/core/logger.py

@@ -1,72 +1,96 @@
-"""Centralized logging configuration for the book downloader application."""
+"""Logging configuration and custom logger with error tracing."""
 
 import logging
 import sys
 from pathlib import Path
 from logging.handlers import RotatingFileHandler
-from env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
 from typing import Any
 
+from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
+
+
 class CustomLogger(logging.Logger):
     """Custom logger class with additional error_trace method."""
-    
+
     def error_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log an error message with full stack trace."""
         self.log_resource_usage()
+        kwargs.pop('exc_info', None)
         self.error(msg, *args, exc_info=True, **kwargs)
 
     def warning_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
         """Log a warning message with full stack trace."""
         self.log_resource_usage()
+        kwargs.pop('exc_info', None)
         self.warning(msg, *args, exc_info=True, **kwargs)
-    
+
     def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log an info message with full stack trace."""
-        self.log_resource_usage()
-        self.info(msg, *args, exc_info=True, **kwargs)
-    
+        """Log an info message (stack trace only if exception active)."""
+        kwargs.pop('exc_info', None)
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.info(msg, *args, exc_info=has_exception, **kwargs)
+
     def debug_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log a debug message with full stack trace."""
-        self.log_resource_usage()
-        self.debug(msg, *args, exc_info=True, **kwargs)
-    
+        """Log a debug message (stack trace only if exception active)."""
+        kwargs.pop('exc_info', None)
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.debug(msg, *args, exc_info=has_exception, **kwargs)
+
     def log_resource_usage(self):
-        import psutil
-        memory = psutil.virtual_memory()
-        available_mb = memory.available / (1024 * 1024)
-        memory_used_mb = memory.used / (1024 * 1024)
-        cpu_percent = psutil.cpu_percent()
-        self.debug(f"Container Memory: Available={available_mb:.2f} MB, Used={memory_used_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
+        # Best-effort only; this should never raise during exception logging.
+        try:
+            import psutil
+
+            # Sum RSS of all processes for actual app memory (container-friendly),
+            # but fall back gracefully on platforms that restrict process enumeration.
+            app_memory_mb = 0.0
+            try:
+                for proc in psutil.process_iter(['memory_info']):
+                    try:
+                        mem = proc.info.get('memory_info')
+                        if mem:
+                            app_memory_mb += mem.rss / (1024 * 1024)
+                    except (psutil.NoSuchProcess, psutil.AccessDenied, KeyError, AttributeError):
+                        continue
+            except (PermissionError, psutil.AccessDenied, OSError):
+                try:
+                    app_memory_mb = psutil.Process().memory_info().rss / (1024 * 1024)
+                except Exception:
+                    app_memory_mb = 0.0
+
+            memory = psutil.virtual_memory()
+            system_used_mb = memory.used / (1024 * 1024)
+            available_mb = memory.available / (1024 * 1024)
+            cpu_percent = psutil.cpu_percent()
+            self.debug(
+                f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, "
+                f"Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%"
+            )
+        except Exception:
+            # Avoid breaking the original log call if psutil is missing or restricted.
+            return
 
 
 def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     """Set up and configure a logger instance.
-    
+
     Args:
         name: The name of the logger instance
         log_file: Optional path to log file. If None, logs only to stdout/stderr
-        
+
     Returns:
         CustomLogger: Configured logger instance with error_trace method
     """
     # Register our custom logger class
     logging.setLoggerClass(CustomLogger)
-    
+
     # Create logger as CustomLogger instance
     logger = CustomLogger(name)
-    log_level = logging.INFO
-    if LOG_LEVEL == "DEBUG":
-        log_level = logging.DEBUG
-    elif LOG_LEVEL == "INFO":
-        log_level = logging.INFO
-    elif LOG_LEVEL == "WARNING":
-        log_level = logging.WARNING
-    elif LOG_LEVEL == "ERROR":
-        log_level = logging.ERROR
-    elif LOG_LEVEL == "CRITICAL":
-        log_level = logging.CRITICAL
+    log_level = getattr(logging, LOG_LEVEL, logging.INFO)
     logger.setLevel(log_level)
-    
+
     formatter = logging.Formatter(
         '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
     )
@@ -77,13 +101,13 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
     console_handler.setLevel(log_level)
     console_handler.addFilter(lambda record: record.levelno < logging.ERROR)  # Only allow logs below ERROR to stdout
     logger.addHandler(console_handler)
-    
+
     # Error handler for stderr
     error_handler = logging.StreamHandler(sys.stderr)
     error_handler.setLevel(logging.ERROR) # Error and above go to stderr
     error_handler.setFormatter(formatter)
     logger.addHandler(error_handler)
-    
+
     # File handler if log file is specified
     try:
         if ENABLE_LOGGING:
@@ -101,4 +125,3 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
         logger.error_trace(f"Failed to create log file: {e}", exc_info=True)
 
     return logger
-

shelfmark/core/mirrors.py

@@ -0,0 +1,256 @@
+"""Centralized mirror configuration for all download sources."""
+
+from typing import List
+
+from shelfmark.core.utils import normalize_http_url
+
+# Lazy import to avoid circular imports
+_config_module = None
+
+
+def _get_config():
+    """Lazy import of config module to avoid circular imports."""
+    global _config_module
+    if _config_module is None:
+        from shelfmark.core.config import config
+        _config_module = config
+    return _config_module
+
+
+# Default mirror lists (hardcoded fallbacks)
+DEFAULT_AA_MIRRORS = [
+    "https://annas-archive.gl",
+    "https://annas-archive.pk",
+    "https://annas-archive.vg",
+    "https://annas-archive.gd",
+]
+
+DEFAULT_LIBGEN_MIRRORS = [
+    "https://libgen.gl",
+    "https://libgen.li",
+    "https://libgen.bz",
+    "https://libgen.la",
+    "https://libgen.vg",
+]
+
+DEFAULT_ZLIB_MIRRORS = [
+    "https://z-lib.fm",
+    "https://z-lib.gs",
+    "https://z-lib.id",
+    "https://z-library.sk",
+    "https://zlibrary-global.se",
+]
+
+DEFAULT_WELIB_MIRRORS = [
+    "https://welib.org",
+]
+
+
+def _normalize_mirror_url(url: str) -> str:
+    return normalize_http_url(url, default_scheme="https")
+
+
+def get_aa_mirrors() -> List[str]:
+    """
+    Get Anna's Archive mirrors.
+
+    Returns:
+        Ordered list of AA mirror URLs.
+
+        If AA_MIRROR_URLS is configured, it is treated as the full list.
+        Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended.
+
+        Notes:
+        - The list is used to populate the AA mirror dropdown in Settings.
+        - When AA_BASE_URL is set to 'auto', mirrors are tried in the order listed.
+    """
+    config = _get_config()
+
+    mirrors: list[str] = []
+
+    configured_list = config.get("AA_MIRROR_URLS", None)
+    if isinstance(configured_list, list):
+        for url in configured_list:
+            normalized = _normalize_mirror_url(str(url))
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+    elif isinstance(configured_list, str) and configured_list.strip():
+        # Allow comma-separated env/manual configs.
+        for url in configured_list.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    if not mirrors:
+        mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
+        mirrors = [url for url in mirrors if url]
+
+        # Backwards-compatible append-only behavior for legacy configs/env.
+        additional = config.get("AA_ADDITIONAL_URLS", "")
+        if additional:
+            for url in additional.split(","):
+                normalized = _normalize_mirror_url(url)
+                if normalized and normalized not in mirrors:
+                    mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_libgen_mirrors() -> List[str]:
+    """
+    Get LibGen mirrors: defaults + any additional from config.
+
+    Returns:
+        List of LibGen mirror URLs (defaults first, then custom additions).
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_mirrors() -> List[str]:
+    """
+    Get Z-Library mirrors, with primary first.
+
+    Returns:
+        List of Z-Library mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_primary_url() -> str:
+    """
+    Get the primary Z-Library mirror URL.
+
+    Returns:
+        Primary Z-Library mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+
+
+def get_zlib_url_template() -> str:
+    """
+    Get Z-Library URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_zlib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_welib_mirrors() -> List[str]:
+    """
+    Get Welib mirrors, with primary first.
+
+    Returns:
+        List of Welib mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_WELIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("WELIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_welib_primary_url() -> str:
+    """
+    Get the primary Welib mirror URL.
+
+    Returns:
+        Primary Welib mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+
+
+def get_welib_url_template() -> str:
+    """
+    Get Welib URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_welib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_zlib_cookie_domains() -> set:
+    """
+    Get set of Z-Library domains that need full cookie handling.
+
+    Used by internal_bypasser for CF bypass cookie management.
+
+    Returns:
+        Set of domain strings (without protocol).
+    """
+    domains = set()
+
+    # Add all default domains
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized:
+            domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+            domains.add(domain)
+
+    # Add custom domains
+    config = _get_config()
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized:
+                domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+                domains.add(domain)
+
+    return domains

shelfmark/core/models.py

@@ -0,0 +1,143 @@
+"""Data structures and models used across the application."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from enum import Enum
+import re
+import time
+
+
+def build_filename(
+    title: str,
+    author: Optional[str] = None,
+    year: Optional[str] = None,
+    fmt: Optional[str] = None,
+) -> str:
+    parts = []
+    if author:
+        parts.append(author)
+        parts.append(" - ")
+    parts.append(title)
+    if year:
+        parts.append(f" ({year})")
+
+    filename = "".join(parts)
+    filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
+
+    if fmt:
+        filename = f"{filename}.{fmt}"
+
+    return filename
+
+
+class QueueStatus(str, Enum):
+    """Enum for possible book queue statuses."""
+    QUEUED = "queued"
+    RESOLVING = "resolving"
+    LOCATING = "locating"
+    DOWNLOADING = "downloading"
+    COMPLETE = "complete"
+    ERROR = "error"
+    CANCELLED = "cancelled"
+
+
+TERMINAL_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset({
+    QueueStatus.COMPLETE, QueueStatus.ERROR, QueueStatus.CANCELLED,
+})
+
+ACTIVE_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset({
+    QueueStatus.QUEUED, QueueStatus.RESOLVING, QueueStatus.LOCATING, QueueStatus.DOWNLOADING,
+})
+
+
+class SearchMode(str, Enum):
+    DIRECT = "direct"
+    UNIVERSAL = "universal"
+
+
+@dataclass
+class QueueItem:
+    """Queue item with priority and metadata."""
+    book_id: str
+    priority: int
+    added_time: float
+
+    def __lt__(self, other):
+        """Compare items for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+
+@dataclass
+class DownloadTask:
+    task_id: str                                # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
+    source: str                                 # Handler name ("direct_download", "prowlarr")
+    title: str                                  # Display title for queue sidebar
+
+    # Display info for queue sidebar
+    author: Optional[str] = None
+    year: Optional[str] = None
+    format: Optional[str] = None
+    size: Optional[str] = None
+    preview: Optional[str] = None
+    content_type: Optional[str] = None  # "book (fiction)", "audiobook", "magazine", etc.
+    source_url: Optional[str] = None  # Original release URL used by source-specific handlers
+
+    # Series info (for library naming templates)
+    series_name: Optional[str] = None
+    series_position: Optional[float] = None  # Float for novellas (e.g., 1.5)
+    subtitle: Optional[str] = None  # Book subtitle for naming templates
+
+    # Hardlinking support
+    original_download_path: Optional[str] = None  # Path in download client (for hardlinking)
+
+    # Search mode - determines post-download processing behavior
+    # See SearchMode enum for behavioral differences
+    search_mode: Optional[SearchMode] = None
+
+    # Output selection for post-processing.
+    # This is captured at queue time so in-flight tasks are not affected if the user changes settings later.
+    output_mode: Optional[str] = None  # e.g. "folder", "booklore", "email"
+    output_args: Dict[str, Any] = field(default_factory=dict)  # Per-output parameters (e.g. email recipient)
+
+    # User association (multi-user support)
+    user_id: Optional[int] = None  # DB user ID who queued this download
+    username: Optional[str] = None  # Username for {User} template variable
+    request_id: Optional[int] = None  # Origin request ID when queued from request fulfilment
+
+    # Runtime state
+    priority: int = 0
+    added_time: float = field(default_factory=time.time)
+    progress: float = 0.0
+    status: QueueStatus = QueueStatus.QUEUED
+    status_message: Optional[str] = None
+    download_path: Optional[str] = None
+    last_error_message: Optional[str] = None
+    last_error_type: Optional[str] = None
+    staged_path: Optional[str] = None
+
+    def __lt__(self, other):
+        """Compare tasks for priority queue (lower priority number = higher precedence)."""
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        return self.added_time < other.added_time
+
+    def get_filename(self) -> str:
+        """Build sanitized filename from task metadata."""
+        if self.download_path:
+            return Path(self.download_path).name
+        return build_filename(self.title, self.author, self.year, self.format)
+
+
+@dataclass
+class SearchFilters:
+    """Filters for book search queries."""
+    isbn: Optional[List[str]] = None
+    author: Optional[List[str]] = None
+    title: Optional[List[str]] = None
+    lang: Optional[List[str]] = None
+    sort: Optional[str] = None
+    content: Optional[List[str]] = None
+    format: Optional[List[str]] = None

shelfmark/core/naming.py

@@ -0,0 +1,249 @@
+"""Template-based naming for library organization."""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional, Union, Mapping
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+# Known variable tokens, sorted longest-first to avoid partial matches
+# e.g., "SeriesPosition" must match before "Series"
+KNOWN_TOKENS = [
+    'seriesposition',
+    'originalname',
+    'partnumber',
+    'subtitle',
+    'author',
+    'series',
+    'title',
+    'year',
+    'user',
+]
+
+# Match any {...} block for template parsing
+BRACE_PATTERN = re.compile(r'\{([^}]+)\}')
+
+# Characters that are invalid in filenames on various filesystems
+INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def _sanitize(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for filesystem use."""
+    if not name:
+        return ""
+
+    sanitized = INVALID_CHARS.sub('_', name)
+    sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized)  # Strip whitespace and dots
+    sanitized = re.sub(r'_+', '_', sanitized)  # Collapse underscores
+    return sanitized[:max_length]
+
+
+def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for use as a filename or path component."""
+    return _sanitize(name, max_length)
+
+
+# Alias for backwards compatibility
+sanitize_path_component = sanitize_filename
+
+
+def format_series_position(position: Optional[Union[str, int, float]]) -> str:
+    if position is None:
+        return ""
+
+    # Display as integer if whole number
+    if isinstance(position, float) and position.is_integer():
+        return str(int(position))
+
+    return str(position)
+
+
+# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
+PAD_NUMBERS_PATTERN = re.compile(r'\d+')
+
+
+def natural_sort_key(path: Union[str, Path]) -> str:
+    """Generate a sort key with padded numbers for natural sorting."""
+    filename = Path(path).name.lower()
+    return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
+
+
+def assign_part_numbers(
+    files: list[Path],
+    zero_pad_width: int = 2,
+) -> list[tuple[Path, str]]:
+    """Sort files naturally and assign sequential part numbers (1, 2, 3...)."""
+    if not files:
+        return []
+
+    sorted_files = sorted(files, key=natural_sort_key)
+    return [
+        (file_path, str(part_num).zfill(zero_pad_width))
+        for part_num, file_path in enumerate(sorted_files, start=1)
+    ]
+
+
+def parse_naming_template(
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    *,
+    allow_path_separators: bool = True,
+) -> str:
+    if not template:
+        return ""
+
+    # Normalize metadata keys to lowercase for case-insensitive matching
+    normalized = {k.lower(): v for k, v in metadata.items()}
+
+    def find_token(content: str) -> tuple[Optional[str], int]:
+        content_lower = content.lower()
+        for token in KNOWN_TOKENS:
+            idx = content_lower.find(token)
+            if idx != -1:
+                return token, idx
+        return None, -1
+
+    def token_value(token: str) -> str:
+        value = normalized.get(token)
+        if token == 'seriesposition':
+            value = format_series_position(value)
+        if value is None:
+            return ""
+        return str(value).strip()
+
+    def render_block(content: str) -> Optional[str]:
+        token, idx = find_token(content)
+        if token is None:
+            return None
+
+        prefix = content[:idx]
+        suffix = content[idx + len(token):]
+        value = token_value(token)
+        if not value:
+            return ""
+
+        if not allow_path_separators:
+            value = value.replace("/", "_")
+        value = sanitize_filename(value)
+        return f"{prefix}{value}{suffix}"
+
+    # Process brace blocks in order so we can support conditional literal blocks like:
+    # { - Part }{PartNumber}
+    matches = list(BRACE_PATTERN.finditer(template))
+    if not matches:
+        result = template
+    else:
+        parts: list[str] = []
+        cursor = 0
+        for idx, match in enumerate(matches):
+            parts.append(template[cursor:match.start()])
+            content = match.group(1)
+            rendered = render_block(content)
+
+            if rendered is not None:
+                parts.append(rendered)
+            else:
+                conditional_literal = False
+                include_literal = False
+                if idx + 1 < len(matches) and match.end() == matches[idx + 1].start():
+                    next_content = matches[idx + 1].group(1)
+                    next_token, _next_idx = find_token(next_content)
+                    if next_token is not None:
+                        conditional_literal = True
+                        include_literal = bool(token_value(next_token))
+                if include_literal:
+                    parts.append(content)
+                elif not conditional_literal:
+                    # Preserve blocks that look like literal text, but treat bare unknown
+                    # placeholders as missing variables.
+                    if re.search(r"\s", content):
+                        parts.append(match.group(0))
+
+            cursor = match.end()
+
+        parts.append(template[cursor:])
+        result = "".join(parts)
+
+    # Clean up any double slashes that might result from empty tokens
+    result = re.sub(r'/+', '/', result)
+
+    # Remove leading/trailing slashes
+    result = result.strip('/')
+
+    # Clean up any orphaned separators (e.g., " - " at start/end, or " -  - ")
+    result = re.sub(r'^[\s\-_.]+', '', result)
+    result = re.sub(r'[\s\-_.]+$', '', result)
+    result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
+
+    # Clean up empty parentheses/brackets
+    result = re.sub(r'\(\s*\)', '', result)
+    result = re.sub(r'\[\s*\]', '', result)
+
+    # Final trim of any trailing separators left after cleanup
+    result = re.sub(r'[\s\-_.]+$', '', result)
+
+    return result
+
+
+def build_library_path(
+    base_path: str,
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    extension: Optional[str] = None,
+) -> Path:
+    relative = parse_naming_template(template, metadata, allow_path_separators=True)
+
+    if not relative:
+        # Fallback to title if template produces empty result
+        title = metadata.get('Title') or metadata.get('title') or 'Unknown'
+        relative = sanitize_filename(str(title))
+
+    # Remove any path traversal attempts
+    relative = relative.replace('..', '')
+
+    base = Path(base_path).resolve()
+    full_path = (base / relative).resolve()
+
+    # Verify the path is within the base directory
+    try:
+        full_path.relative_to(base)
+    except ValueError:
+        raise ValueError(f"Path traversal detected: template would escape library directory")
+
+    if extension:
+        ext = extension.lstrip('.')
+        # Don't use with_suffix() - it replaces everything after the first dot
+        # e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
+        full_path = Path(f"{full_path}.{ext}")
+
+    return full_path
+
+
+def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
+    """Check if two paths are on the same filesystem."""
+    path1 = Path(path1)
+    path2 = Path(path2)
+
+    def get_device(p: Path) -> Optional[int]:
+        try:
+            while not p.exists():
+                p = p.parent
+                if p == p.parent:
+                    break
+            return os.stat(p).st_dev
+        except (OSError, PermissionError) as e:
+            logger.debug(f"Cannot stat {p}: {e}")
+            return None
+
+    dev1 = get_device(path1)
+    dev2 = get_device(path2)
+
+    if dev1 is None or dev2 is None:
+        logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
+        return False
+
+    return dev1 == dev2

shelfmark/core/notifications.py

@@ -0,0 +1,613 @@
+"""Apprise notification dispatch for global and per-user events."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Iterable, Iterator
+from urllib.parse import urlsplit
+
+try:
+    import apprise
+except Exception:  # pragma: no cover - exercised in tests via monkeypatch
+    apprise = None  # type: ignore[assignment]
+
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Small pool for non-blocking dispatch. Notification sends are I/O bound and infrequent.
+_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="Notify")
+_ROUTE_EVENT_ALL = "all"
+_APPRISE_APP_ID = "Shelfmark"
+_APPRISE_APP_DESC = "Shelfmark notifications"
+_APPRISE_LOGO_URL = (
+    "https://raw.githubusercontent.com/calibrain/shelfmark/main/src/frontend/public/logo.png"
+)
+_APPRISE_LOGGER_NAME = "apprise"
+
+
+class NotificationEvent(str, Enum):
+    """Global notification event identifiers."""
+
+    REQUEST_CREATED = "request_created"
+    REQUEST_FULFILLED = "request_fulfilled"
+    REQUEST_REJECTED = "request_rejected"
+    DOWNLOAD_COMPLETE = "download_complete"
+    DOWNLOAD_FAILED = "download_failed"
+
+
+@dataclass
+class NotificationContext:
+    """Context used to render notification templates."""
+
+    event: NotificationEvent
+    title: str
+    author: str
+    username: str | None = None
+    content_type: str | None = None
+    format: str | None = None
+    source: str | None = None
+    admin_note: str | None = None
+    error_message: str | None = None
+
+
+def _normalize_urls(value: Any) -> list[str]:
+    if value is None:
+        return []
+
+    raw_values: list[Any]
+    if isinstance(value, list):
+        raw_values = value
+    elif isinstance(value, str):
+        # Support legacy/manual configs.
+        raw_values = [segment for part in value.splitlines() for segment in part.split(",")]
+    else:
+        raw_values = [value]
+
+    normalized: list[str] = []
+    seen: set[str] = set()
+    for raw_url in raw_values:
+        url = str(raw_url or "").strip()
+        if not url:
+            continue
+        # Strip invisible/non-ASCII characters that can sneak in via copy-paste
+        # (zero-width spaces, smart quotes, non-breaking spaces, etc.).
+        # These pass Apprise URL validation but cause UnicodeEncodeError when
+        # requests tries to latin-1 encode credentials for Basic Auth headers.
+        url = url.encode("ascii", errors="ignore").decode("ascii").strip()
+        if not url:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        normalized.append(url)
+    return normalized
+
+
+def _extract_url_schemes(urls: Iterable[str]) -> list[str]:
+    schemes: list[str] = []
+    seen: set[str] = set()
+    for raw_url in urls:
+        scheme = urlsplit(str(raw_url or "")).scheme.lower()
+        if not scheme or scheme in seen:
+            continue
+        seen.add(scheme)
+        schemes.append(scheme)
+    return schemes
+
+
+class _AppriseLogCapture(logging.Handler):
+    def __init__(self, *, thread_id: int):
+        super().__init__(level=logging.INFO)
+        self.records: list[tuple[int, str, str, str]] = []
+        self._thread_id = thread_id
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.thread != self._thread_id:
+            return
+
+        message = record.getMessage()
+        if message:
+            exception_summary = ""
+            if record.exc_info and record.exc_info[0]:
+                exc_type = getattr(record.exc_info[0], "__name__", "Exception")
+                exc = record.exc_info[1]
+                exception_summary = f"{exc_type}: {exc}"
+            elif record.exc_text:
+                exception_summary = str(record.exc_text).strip()
+
+            self.records.append((record.levelno, record.name, str(message), exception_summary))
+
+
+@contextmanager
+def _capture_apprise_logs(*, min_level: int = logging.INFO) -> Iterator[list[tuple[int, str, str, str]]]:
+    apprise_logger = logging.getLogger(_APPRISE_LOGGER_NAME)
+    previous_level = apprise_logger.level
+    handler = _AppriseLogCapture(thread_id=threading.get_ident())
+    apprise_logger.addHandler(handler)
+
+    if previous_level == logging.NOTSET or previous_level > min_level:
+        apprise_logger.setLevel(min_level)
+
+    try:
+        yield handler.records
+    finally:
+        apprise_logger.removeHandler(handler)
+        apprise_logger.setLevel(previous_level)
+
+
+def _log_apprise_records(records: Iterable[tuple[int, str, str, str]]) -> None:
+    seen: set[tuple[int, str, str, str]] = set()
+    for level, source, raw_message, raw_exception_summary in records:
+        message = str(raw_message or "").strip()
+        source_name = str(source or "").strip() or _APPRISE_LOGGER_NAME
+        exception_summary = str(raw_exception_summary or "").strip()
+        key = (int(level), source_name, message, exception_summary)
+        if not message or key in seen:
+            continue
+        seen.add(key)
+
+        full_message = message if not exception_summary else f"{message} ({exception_summary})"
+
+        if level >= logging.ERROR:
+            logger.error("Apprise source [%s]: %s", source_name, full_message)
+        elif level >= logging.WARNING:
+            logger.warning("Apprise source [%s]: %s", source_name, full_message)
+        else:
+            logger.info("Apprise source [%s]: %s", source_name, full_message)
+
+
+def _log_apprise_exception_debug(*, action: str, scheme: str, exc: Exception) -> None:
+    logger.debug(
+        "Apprise %s raised %s for scheme '%s': %s",
+        action,
+        type(exc).__name__,
+        scheme,
+        exc,
+        exc_info=True,
+    )
+
+
+def _build_apprise_warning_detail(
+    records: Iterable[tuple[int, str, str, str]],
+    *,
+    scheme: str,
+) -> str | None:
+    for level, source, raw_message, raw_exception_summary in records:
+        if level < logging.WARNING:
+            continue
+
+        message = str(raw_message or "").strip()
+        if not message:
+            continue
+
+        source_name = str(source or "").strip()
+        exception_summary = str(raw_exception_summary or "").strip()
+        full_message = message if not exception_summary else f"{message} ({exception_summary})"
+
+        if source_name and source_name != _APPRISE_LOGGER_NAME:
+            return f"{scheme}: {source_name}: {full_message}"
+        return f"{scheme}: {full_message}"
+    return None
+
+
+def _normalize_routes(value: Any) -> list[dict[str, str]]:
+    if not isinstance(value, list):
+        return []
+
+    allowed_events = {_ROUTE_EVENT_ALL, *(event.value for event in NotificationEvent)}
+    normalized: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        raw_events = row.get("event")
+        if isinstance(raw_events, list):
+            event_values = raw_events
+        elif isinstance(raw_events, (tuple, set)):
+            event_values = list(raw_events)
+        else:
+            event_values = [raw_events]
+
+        url = str(row.get("url") or "").strip()
+        if not url:
+            continue
+
+        row_events: list[str] = []
+        for raw_event in event_values:
+            event = str(raw_event or "").strip().lower()
+            if event not in allowed_events:
+                continue
+            if event in row_events:
+                continue
+            row_events.append(event)
+
+        if _ROUTE_EVENT_ALL in row_events:
+            row_events = [_ROUTE_EVENT_ALL]
+
+        for event in row_events:
+            key = (event, url)
+            if key in seen:
+                continue
+            seen.add(key)
+
+            normalized.append({"event": event, "url": url})
+
+    return normalized
+
+
+def _resolve_admin_routes() -> list[dict[str, str]]:
+    return _normalize_routes(app_config.get("ADMIN_NOTIFICATION_ROUTES", []))
+
+
+def _normalize_user_id(value: Any) -> int | None:
+    try:
+        user_id = int(value)
+    except (TypeError, ValueError):
+        return None
+    if user_id < 1:
+        return None
+    return user_id
+
+
+def _resolve_user_routes(user_id: int | None) -> list[dict[str, str]]:
+    normalized_user_id = _normalize_user_id(user_id)
+    if normalized_user_id is None:
+        return []
+
+    return _normalize_routes(
+        app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=normalized_user_id)
+    )
+
+
+def _resolve_route_urls_for_event(
+    routes: list[dict[str, str]],
+    event: NotificationEvent,
+) -> list[str]:
+    selected: list[str] = []
+    seen: set[str] = set()
+    event_value = event.value
+
+    for row in routes:
+        row_event = row.get("event", "")
+        if row_event not in {_ROUTE_EVENT_ALL, event_value}:
+            continue
+
+        url = row.get("url", "")
+        if not url or url in seen:
+            continue
+
+        seen.add(url)
+        selected.append(url)
+
+    return selected
+
+
+def _resolve_notify_type(event: NotificationEvent) -> Any:
+    if apprise is None:
+        fallback = {
+            NotificationEvent.REQUEST_CREATED: "info",
+            NotificationEvent.REQUEST_FULFILLED: "success",
+            NotificationEvent.REQUEST_REJECTED: "warning",
+            NotificationEvent.DOWNLOAD_COMPLETE: "success",
+            NotificationEvent.DOWNLOAD_FAILED: "failure",
+        }
+        return fallback[event]
+
+    mapping = {
+        NotificationEvent.REQUEST_CREATED: apprise.NotifyType.INFO,
+        NotificationEvent.REQUEST_FULFILLED: apprise.NotifyType.SUCCESS,
+        NotificationEvent.REQUEST_REJECTED: apprise.NotifyType.WARNING,
+        NotificationEvent.DOWNLOAD_COMPLETE: apprise.NotifyType.SUCCESS,
+        NotificationEvent.DOWNLOAD_FAILED: apprise.NotifyType.FAILURE,
+    }
+    return mapping[event]
+
+
+def _clean_text(value: Any, fallback: str) -> str:
+    text = str(value or "").strip()
+    return text or fallback
+
+
+def _render_message(context: NotificationContext) -> tuple[str, str]:
+    event = context.event
+    title = _clean_text(context.title, "Unknown title")
+    author = _clean_text(context.author, "Unknown author")
+    username = _clean_text(context.username, "A user")
+
+    if event == NotificationEvent.REQUEST_CREATED:
+        return "New Request", f'{username} requested "{title}" by {author}'
+    if event == NotificationEvent.REQUEST_FULFILLED:
+        return "Request Approved", f'Request for "{title}" by {author} was approved.'
+    if event == NotificationEvent.REQUEST_REJECTED:
+        note = _clean_text(context.admin_note, "")
+        note_line = f"\nNote: {note}" if note else ""
+        return "Request Rejected", f'Request for "{title}" by {author} was rejected.{note_line}'
+    if event == NotificationEvent.DOWNLOAD_COMPLETE:
+        return "Download Complete", f'"{title}" by {author} downloaded successfully.'
+
+    error_message = _clean_text(context.error_message, "")
+    error_line = f"\nError: {error_message}" if error_message else ""
+    return "Download Failed", f'Failed to download "{title}" by {author}.{error_line}'
+
+
+def _plugin_label(plugin: Any, fallback_scheme: str) -> str:
+    """Build a human-readable label from a validated Apprise plugin.
+
+    Combines the URL scheme with the plugin's service name (app_id) and
+    privacy-safe URL for richer diagnostics, e.g.
+    ``"slack (Slack - slack://TokenA/To...n/To...n/)"``
+    """
+    parts: list[str] = [fallback_scheme]
+
+    app_id = getattr(plugin, "app_id", None)
+    if app_id and str(app_id) != fallback_scheme:
+        privacy_url: str | None = None
+        try:
+            privacy_url = plugin.url(privacy=True)
+        except Exception:
+            pass
+
+        suffix = str(app_id)
+        if privacy_url:
+            suffix = f"{suffix} - {privacy_url}"
+        parts.append(f"({suffix})")
+
+    return " ".join(parts)
+
+
+def _dispatch_to_apprise(
+    urls: Iterable[str],
+    *,
+    title: str,
+    body: str,
+    notify_type: Any,
+) -> dict[str, Any]:
+    normalized_urls = _normalize_urls(list(urls))
+    url_schemes = _extract_url_schemes(normalized_urls)
+    if not normalized_urls:
+        return {"success": False, "message": "No notification URLs configured"}
+
+    if apprise is None:
+        return {"success": False, "message": "Apprise is not installed"}
+
+    valid_urls = 0
+    invalid_urls = 0
+    delivered_urls = 0
+    failed_delivery_urls = 0
+    failure_details: list[str] = []
+
+    for url in normalized_urls:
+        scheme = urlsplit(url).scheme or "unknown"
+        apobj = _create_apprise_client()
+        if apobj is None:
+            return {"success": False, "message": "Apprise is not installed"}
+
+        registration_failure_detail: str | None = None
+        with _capture_apprise_logs(min_level=logging.INFO) as apprise_records:
+            try:
+                plugin = apprise.Apprise.instantiate(url, asset=getattr(apobj, "asset", None))
+            except Exception as exc:
+                logger.warning(
+                    "Failed to register notification route URL for scheme '%s': %s",
+                    scheme,
+                    exc,
+                )
+                _log_apprise_exception_debug(
+                    action="route registration",
+                    scheme=scheme,
+                    exc=exc,
+                )
+                registration_failure_detail = (
+                    f"{scheme}: route registration failed ({type(exc).__name__}: {exc})"
+                )
+                failure_details.append(registration_failure_detail)
+                plugin = None
+
+            if plugin is None:
+                invalid_urls += 1
+                logger.warning("Apprise rejected notification route URL for scheme '%s'", scheme)
+                _log_apprise_records(apprise_records)
+                warning_detail = _build_apprise_warning_detail(apprise_records, scheme=scheme)
+                if warning_detail:
+                    failure_details.append(warning_detail)
+                elif registration_failure_detail is None:
+                    failure_details.append(f"{scheme}: route URL rejected by Apprise")
+                continue
+
+            plugin_label = _plugin_label(plugin, scheme)
+            apobj.add(plugin)
+            valid_urls += 1
+
+            try:
+                delivered = bool(apobj.notify(title=title, body=body, notify_type=notify_type))
+            except Exception as exc:
+                _log_apprise_records(apprise_records)
+                failed_delivery_urls += 1
+                logger.warning(
+                    "Apprise notify raised %s for %s: %s",
+                    type(exc).__name__,
+                    plugin_label,
+                    exc,
+                )
+                _log_apprise_exception_debug(action="notify", scheme=scheme, exc=exc)
+                warning_detail = _build_apprise_warning_detail(apprise_records, scheme=scheme)
+                if warning_detail:
+                    failure_details.append(warning_detail)
+                else:
+                    failure_details.append(
+                        f"{scheme}: notify raised {type(exc).__name__}: {exc}"
+                    )
+                continue
+
+        _log_apprise_records(apprise_records)
+        if delivered:
+            delivered_urls += 1
+            logger.debug("Notification delivered via %s", plugin_label)
+            continue
+
+        failed_delivery_urls += 1
+        logger.warning("Apprise notify returned False for %s", plugin_label)
+        warning_detail = _build_apprise_warning_detail(apprise_records, scheme=scheme)
+        if warning_detail:
+            failure_details.append(warning_detail)
+        else:
+            failure_details.append(f"{scheme}: delivery failed")
+
+    scheme_summary = ", ".join(url_schemes) if url_schemes else "unknown"
+    if valid_urls == 0:
+        logger.warning(
+            "No valid Apprise notification routes after registration for scheme(s): %s",
+            scheme_summary,
+        )
+        result: dict[str, Any] = {
+            "success": False,
+            "message": "No valid notification URLs configured",
+        }
+        if failure_details:
+            result["details"] = failure_details
+        return result
+
+    if delivered_urls == 0:
+        logger.warning(
+            (
+                "Apprise notify returned False for scheme(s): %s "
+                "(valid_urls=%s invalid_urls=%s failed_deliveries=%s)"
+            ),
+            scheme_summary,
+            valid_urls,
+            invalid_urls,
+            failed_delivery_urls,
+        )
+        result = {"success": False, "message": "Notification delivery failed"}
+        if failure_details:
+            result["details"] = failure_details
+        return result
+
+    message = f"Notification sent to {delivered_urls} URL(s)"
+    failed_urls = invalid_urls + failed_delivery_urls
+    if failed_urls:
+        message += f" ({failed_urls} URL(s) failed)"
+    result = {"success": True, "message": message}
+    if failure_details:
+        result["details"] = failure_details
+    return result
+
+
+def _create_apprise_client() -> Any:
+    if apprise is None:
+        return None
+
+    apprise_cls = getattr(apprise, "Apprise", None)
+    if apprise_cls is None:
+        return None
+
+    apprise_asset_cls = getattr(apprise, "AppriseAsset", None)
+    if apprise_asset_cls is None:
+        return apprise_cls()
+
+    try:
+        asset = apprise_asset_cls(
+            app_id=_APPRISE_APP_ID,
+            app_desc=_APPRISE_APP_DESC,
+            image_url_logo=_APPRISE_LOGO_URL,
+        )
+    except TypeError:
+        # Support older Apprise versions that do not expose image_url_logo.
+        asset = apprise_asset_cls(
+            app_id=_APPRISE_APP_ID,
+            app_desc=_APPRISE_APP_DESC,
+        )
+    except Exception:
+        return apprise_cls()
+
+    try:
+        return apprise_cls(asset=asset)
+    except Exception:
+        return apprise_cls()
+
+
+def _send_admin_event(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> dict[str, Any]:
+    title, body = _render_message(context)
+    notify_type = _resolve_notify_type(event)
+    return _dispatch_to_apprise(urls, title=title, body=body, notify_type=notify_type)
+
+
+def notify_admin(event: NotificationEvent, context: NotificationContext) -> None:
+    """Send a global admin notification for an event if subscribed."""
+    routes = _resolve_admin_routes()
+    urls = _resolve_route_urls_for_event(routes, event)
+    if not urls:
+        return
+
+    try:
+        _executor.submit(_dispatch_admin_async, event, context, urls)
+    except Exception as exc:
+        logger.warning("Failed to queue admin notification '%s': %s", event.value, exc)
+
+
+def notify_user(user_id: int | None, event: NotificationEvent, context: NotificationContext) -> None:
+    """Send a per-user notification for an event if subscribed."""
+    normalized_user_id = _normalize_user_id(user_id)
+    if normalized_user_id is None:
+        return
+
+    routes = _resolve_user_routes(normalized_user_id)
+    urls = _resolve_route_urls_for_event(routes, event)
+    if not urls:
+        return
+
+    try:
+        _executor.submit(_dispatch_user_async, normalized_user_id, event, context, urls)
+    except Exception as exc:
+        logger.warning(
+            "Failed to queue user notification '%s' for user_id=%s: %s",
+            event.value,
+            normalized_user_id,
+            exc,
+        )
+
+
+def _dispatch_admin_async(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> None:
+    result = _send_admin_event(event, context, urls)
+    if not result.get("success", False):
+        logger.warning("Admin notification failed for event '%s': %s", event.value, result.get("message"))
+
+
+def _dispatch_user_async(
+    user_id: int,
+    event: NotificationEvent,
+    context: NotificationContext,
+    urls: list[str],
+) -> None:
+    result = _send_admin_event(event, context, urls)
+    if not result.get("success", False):
+        logger.warning(
+            "User notification failed for event '%s' (user_id=%s): %s",
+            event.value,
+            user_id,
+            result.get("message"),
+        )
+
+
+def send_test_notification(urls: list[str]) -> dict[str, Any]:
+    """Send a synchronous test notification to the provided URLs."""
+    normalized_urls = _normalize_urls(urls)
+    if not normalized_urls:
+        return {"success": False, "message": "No notification URLs configured"}
+
+    test_context = NotificationContext(
+        event=NotificationEvent.REQUEST_CREATED,
+        title="Shelfmark Test Notification",
+        author="Shelfmark",
+        username="Shelfmark",
+    )
+    return _send_admin_event(NotificationEvent.REQUEST_CREATED, test_context, normalized_urls)

shelfmark/core/oidc_auth.py

@@ -0,0 +1,80 @@
+"""OIDC authentication helpers.
+
+Handles group claim parsing, user info extraction, and user provisioning.
+Flask route handlers are registered separately in main.py.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.external_user_linking import upsert_external_user
+from shelfmark.core.user_db import UserDB
+
+def parse_group_claims(id_token: Dict[str, Any], group_claim: str) -> List[str]:
+    """Extract group list from an ID token claim.
+
+    Supports list, comma-separated string, or pipe-separated string.
+    Returns empty list if claim is missing.
+    """
+    raw = id_token.get(group_claim)
+    if raw is None:
+        return []
+    if isinstance(raw, list):
+        return [str(g).strip() for g in raw if str(g).strip()]
+    if isinstance(raw, str):
+        delimiter = "," if "," in raw else "|"
+        return [g.strip() for g in raw.split(delimiter) if g.strip()]
+    return []
+
+
+def extract_user_info(id_token: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract user info from OIDC ID token claims.
+
+    Returns a dict with keys: oidc_subject, username, email, display_name.
+    Falls back through preferred_username -> email -> sub for username.
+    """
+    sub = id_token.get("sub", "")
+    email = id_token.get("email")
+    display_name = id_token.get("name")
+    username = id_token.get("preferred_username") or email or sub
+
+    return {
+        "oidc_subject": sub,
+        "username": username,
+        "email": email,
+        "display_name": display_name,
+    }
+
+
+def provision_oidc_user(
+    db: UserDB,
+    user_info: Dict[str, Any],
+    is_admin: Optional[bool] = None,
+    allow_email_link: bool = False,
+    allow_create: bool = True,
+) -> Optional[Dict[str, Any]]:
+    """Create or update a user from OIDC claims.
+
+    Matching and collision handling use the shared external user linker:
+    - OIDC subject first
+    - optionally unique email linking (when `allow_email_link=True`)
+    - username conflict resolution via numeric suffix.
+
+    Returns None when no existing user is matchable and `allow_create=False`.
+    """
+    oidc_subject = user_info["oidc_subject"]
+    user, _ = upsert_external_user(
+        db,
+        auth_source="oidc",
+        username=user_info["username"] or oidc_subject,
+        role="admin" if is_admin else "user",
+        email=user_info.get("email"),
+        display_name=user_info.get("display_name"),
+        subject_field="oidc_subject",
+        subject=oidc_subject,
+        allow_email_link=allow_email_link,
+        sync_role=is_admin is not None,
+        allow_create=allow_create,
+        collision_strategy="suffix",
+        context="oidc_login",
+    )
+    return user

shelfmark/core/oidc_routes.py

@@ -0,0 +1,223 @@
+"""OIDC Flask route handlers using Authlib.
+
+Registers /api/auth/oidc/login and /api/auth/oidc/callback endpoints.
+Business logic remains in oidc_auth.py.
+"""
+
+from typing import Any
+from urllib.parse import quote
+
+from authlib.jose.errors import InvalidClaimError
+from authlib.integrations.flask_client import OAuth
+from flask import Flask, jsonify, redirect, request, session
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.oidc_auth import (
+    extract_user_info,
+    parse_group_claims,
+    provision_oidc_user,
+)
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_db import UserDB
+from shelfmark.download.network import get_ssl_verify
+
+logger = setup_logger(__name__)
+oauth = OAuth()
+
+
+def _normalize_claims(raw_claims: Any) -> dict[str, Any]:
+    """Return a plain dict for claims from Authlib token/userinfo payloads."""
+    if raw_claims is None:
+        return {}
+    if isinstance(raw_claims, dict):
+        return raw_claims
+    if hasattr(raw_claims, "to_dict"):
+        return raw_claims.to_dict()  # type: ignore[no-any-return]
+    try:
+        return dict(raw_claims)
+    except Exception:
+        return {}
+
+
+def _has_username_or_email(claims: dict[str, Any]) -> bool:
+    """Return True when claims include a usable username or email."""
+    for key in ("preferred_username", "email"):
+        value = claims.get(key)
+        if isinstance(value, str) and value.strip():
+            return True
+    return False
+
+
+def _login_error_url(message: str) -> str:
+    """Build a login URL (with script_root) that includes an OIDC error message."""
+    script_root = request.script_root.rstrip("/")
+    login_url = f"{script_root}/login" if script_root else "/login"
+    return f"{login_url}?oidc_error={quote(message)}"
+
+
+def _get_oidc_client() -> tuple[Any, dict[str, Any]]:
+    """Register and return an OIDC client from the current security config."""
+    config = load_config_file("security")
+    discovery_url = config.get("OIDC_DISCOVERY_URL", "")
+    client_id = config.get("OIDC_CLIENT_ID", "")
+
+    if not discovery_url or not client_id:
+        raise ValueError("OIDC not configured")
+
+    configured_scopes = config.get("OIDC_SCOPES", ["openid", "email", "profile"])
+    if isinstance(configured_scopes, list):
+        scope_values = [str(scope).strip() for scope in configured_scopes if str(scope).strip()]
+    elif isinstance(configured_scopes, str):
+        delimiter = "," if "," in configured_scopes else " "
+        scope_values = [scope.strip() for scope in configured_scopes.split(delimiter) if scope.strip()]
+    else:
+        scope_values = []
+
+    scopes = list(dict.fromkeys(["openid"] + scope_values))
+
+    admin_group = config.get("OIDC_ADMIN_GROUP", "")
+    group_claim = config.get("OIDC_GROUP_CLAIM", "groups")
+    use_admin_group = config.get("OIDC_USE_ADMIN_GROUP", True)
+    if admin_group and use_admin_group and group_claim and group_claim not in scopes:
+        scopes.append(group_claim)
+
+    def _ssl_compliance_fix(session, **kwargs):
+        """Set session.verify based on the Certificate Validation setting."""
+        session.verify = get_ssl_verify(discovery_url)
+        return session
+
+    oauth._clients.pop("shelfmark_idp", None)
+    oauth.register(
+        name="shelfmark_idp",
+        client_id=client_id,
+        client_secret=config.get("OIDC_CLIENT_SECRET", ""),
+        server_metadata_url=discovery_url,
+        client_kwargs={
+            "scope": " ".join(scopes),
+            "code_challenge_method": "S256",
+        },
+        compliance_fix=_ssl_compliance_fix,
+        overwrite=True,
+    )
+
+    client = oauth.create_client("shelfmark_idp")
+    if client is None:
+        raise RuntimeError("OIDC client initialization failed")
+
+    return client, config
+
+
+def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
+    """Register OIDC authentication routes on the Flask app."""
+    oauth.init_app(app)
+
+    @app.route("/api/auth/oidc/login", methods=["GET"])
+    def oidc_login():
+        """Initiate OIDC login flow and redirect to the provider."""
+        try:
+            client, _ = _get_oidc_client()
+            redirect_uri = request.url_root.rstrip("/") + "/api/auth/oidc/callback"
+            return client.authorize_redirect(redirect_uri)
+        except ValueError:
+            return jsonify({"error": "OIDC not configured"}), 500
+        except Exception as e:
+            logger.error(f"OIDC login error: {e}")
+            return jsonify({"error": "OIDC login failed"}), 500
+
+    @app.route("/api/auth/oidc/callback", methods=["GET"])
+    def oidc_callback():
+        """Handle OIDC callback from identity provider."""
+        try:
+            error = request.args.get("error")
+            if error:
+                logger.warning(f"OIDC callback error from IdP: {error}")
+                return redirect(_login_error_url("Authentication failed"))
+
+            client, config = _get_oidc_client()
+            try:
+                token = client.authorize_access_token()
+            except InvalidClaimError as e:
+                claim_name = getattr(e, "claim_name", "unknown")
+                discovery_url = str(config.get("OIDC_DISCOVERY_URL", ""))
+                provider_issuer = ""
+                try:
+                    metadata = client.load_server_metadata()
+                    if isinstance(metadata, dict):
+                        provider_issuer = str(metadata.get("issuer", ""))
+                except Exception as metadata_error:
+                    logger.debug(f"OIDC metadata lookup failed during claim diagnostics: {metadata_error}")
+
+                logger.error(
+                    "OIDC callback claim validation failed: claim=%s error=%s discovery_url=%s provider_issuer=%s",
+                    claim_name,
+                    e,
+                    discovery_url or "<unset>",
+                    provider_issuer or "<unknown>",
+                )
+                if claim_name == "iss":
+                    msg = (
+                        "OIDC issuer validation failed. Verify your discovery URL and IdP issuer/"
+                        "external URL configuration."
+                    )
+                    return redirect(_login_error_url(msg))
+
+                return redirect(_login_error_url(f"OIDC token claim validation failed: {claim_name}"))
+            claims = _normalize_claims(token.get("userinfo"))
+
+            # If userinfo is missing or claims are too sparse, request it explicitly.
+            if not claims or not _has_username_or_email(claims):
+                fetched_claims: dict[str, Any] = {}
+                try:
+                    fetched_claims = _normalize_claims(client.userinfo(token=token))
+                except TypeError:
+                    fetched_claims = _normalize_claims(client.userinfo())
+                except Exception as e:
+                    logger.error(f"Failed to fetch OIDC userinfo: {e}")
+                if fetched_claims:
+                    claims = {**claims, **fetched_claims}
+
+            if not claims:
+                msg = "OIDC authentication failed: missing user claims"
+                logger.error(msg)
+                return redirect(_login_error_url(msg))
+
+            group_claim = config.get("OIDC_GROUP_CLAIM", "groups")
+            admin_group = config.get("OIDC_ADMIN_GROUP", "")
+            use_admin_group = config.get("OIDC_USE_ADMIN_GROUP", True)
+            auto_provision = config.get("OIDC_AUTO_PROVISION", True)
+
+            user_info = extract_user_info(claims)
+            groups = parse_group_claims(claims, group_claim)
+
+            is_admin = None
+            if admin_group and use_admin_group:
+                is_admin = admin_group in groups
+
+            allow_email_link = bool(user_info.get("email"))
+            user = provision_oidc_user(
+                user_db,
+                user_info,
+                is_admin=is_admin,
+                allow_email_link=allow_email_link,
+                allow_create=bool(auto_provision),
+            )
+            if user is None:
+                logger.warning(
+                    f"OIDC login rejected: auto-provision disabled for {user_info['username']}"
+                )
+                return redirect(_login_error_url("Account not found. Contact your administrator."))
+
+            session["user_id"] = user["username"]
+            session["is_admin"] = user.get("role") == "admin"
+            session["db_user_id"] = user["id"]
+            session.permanent = True
+
+            logger.info(f"OIDC login successful: {user['username']} (admin={is_admin})")
+            return redirect(request.script_root or "/")
+
+        except ValueError as e:
+            logger.error(f"OIDC callback error: {e}")
+            return redirect(_login_error_url(str(e)))
+        except Exception as e:
+            logger.error(f"OIDC callback error: {e}")
+            return redirect(_login_error_url("Authentication failed"))

shelfmark/core/onboarding.py

@@ -0,0 +1,435 @@
+"""
+Onboarding wizard configuration.
+
+Defines the steps and fields for the first-run onboarding experience.
+Reuses field definitions from the settings registry where possible.
+"""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    HeadingField,
+    SettingsField,
+    get_settings_tab,
+    serialize_field,
+    save_config_file,
+    get_setting_value,
+)
+
+logger = setup_logger(__name__)
+
+
+ONBOARDING_STORAGE_KEY = "onboarding_complete"
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def is_onboarding_complete() -> bool:
+    """Check if onboarding has been completed."""
+    from shelfmark.config.env import ONBOARDING
+
+    # If onboarding is disabled via env var, treat as complete
+    if not ONBOARDING:
+        return True
+
+    config_file = _get_config_dir() / "settings.json"
+    if not config_file.exists():
+        return False
+
+    try:
+        with open(config_file, 'r') as f:
+            config = json.load(f)
+            return config.get(ONBOARDING_STORAGE_KEY, False)
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning(f"Could not read onboarding status from settings.json: {e}")
+        return False
+
+
+def mark_onboarding_complete() -> bool:
+    """Mark onboarding as complete."""
+    try:
+        return save_config_file("general", {ONBOARDING_STORAGE_KEY: True})
+    except Exception as e:
+        logger.error(f"Failed to mark onboarding complete: {e}")
+        return False
+
+
+def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField]:
+    """
+    Extract a specific field from a registered settings tab.
+
+    Args:
+        tab_name: Name of the settings tab (e.g., 'search_mode', 'hardcover')
+        field_key: Key of the field to extract (e.g., 'SEARCH_MODE', 'HARDCOVER_API_KEY')
+
+    Returns:
+        The field if found, None otherwise
+    """
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        logger.warning(f"Settings tab not found: {tab_name}")
+        return None
+
+    for field in tab.fields:
+        if hasattr(field, 'key') and field.key == field_key:
+            return field
+
+    logger.warning(f"Field {field_key} not found in tab {tab_name}")
+    return None
+
+
+def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
+    """
+    Clone a field with optional attribute overrides.
+
+    Useful for customizing labels, descriptions, or defaults for onboarding context.
+    """
+    return replace(field, **overrides)
+
+
+# =============================================================================
+# Step Definitions
+# =============================================================================
+
+
+def get_search_mode_fields() -> List[SettingsField]:
+    """Step 1: Choose search mode - uses actual SEARCH_MODE field from settings."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="welcome_heading",
+            title="Welcome to Shelfmark",
+            description="Let's configure how you want to search for and download books.",
+        ),
+    ]
+
+    # Get the actual SEARCH_MODE field from settings
+    search_mode_field = _get_field_from_tab("search_mode", "SEARCH_MODE")
+    if search_mode_field:
+        # Clone with onboarding-specific description
+        fields.append(_clone_field_with_overrides(
+            search_mode_field,
+            description="Choose how you want to find books.",
+        ))
+
+    return fields
+
+
+def get_metadata_provider_fields() -> List[SettingsField]:
+    """Step 2: Choose metadata provider - uses actual METADATA_PROVIDER field."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="metadata_heading",
+            title="Metadata Provider",
+            description="Choose where to search for book information. You can enable more providers in Settings later.",
+        ),
+    ]
+
+    # Get the actual METADATA_PROVIDER field from settings
+    provider_field = _get_field_from_tab("search_mode", "METADATA_PROVIDER")
+    if provider_field:
+        # Custom options with Hardcover marked as recommended
+        onboarding_options = [
+            {
+                "value": "hardcover",
+                "label": "Hardcover (Recommended)",
+                "description": "Modern book tracking platform with excellent metadata, ratings, and series information. Requires free API key.",
+            },
+            {
+                "value": "openlibrary",
+                "label": "Open Library",
+                "description": "Free, open-source library catalog from the Internet Archive. No API key required.",
+            },
+            {
+                "value": "googlebooks",
+                "label": "Google Books",
+                "description": "Google's book database with good coverage. Requires free API key.",
+            },
+        ]
+
+        # Clone with onboarding-specific options and default
+        fields.append(_clone_field_with_overrides(
+            provider_field,
+            default="hardcover",
+            options=onboarding_options,
+        ))
+
+    return fields
+
+
+def get_hardcover_setup_fields() -> List[SettingsField]:
+    """Step 3a: Configure Hardcover - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="hardcover_setup_heading",
+            title="Hardcover Setup",
+            description="Get your free API key from hardcover.app/account/api",
+            link_url="https://hardcover.app/account/api",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual HARDCOVER_API_KEY field
+    api_key_field = _get_field_from_tab("hardcover", "HARDCOVER_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("hardcover", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_googlebooks_setup_fields() -> List[SettingsField]:
+    """Step 3b: Configure Google Books - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="googlebooks_setup_heading",
+            title="Google Books Setup",
+            description="Get your free API key from Google Cloud Console (APIs & Services > Credentials).",
+            link_url="https://console.cloud.google.com/apis/library/books.googleapis.com",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual GOOGLEBOOKS_API_KEY field
+    api_key_field = _get_field_from_tab("googlebooks", "GOOGLEBOOKS_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("googlebooks", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_prowlarr_fields() -> List[SettingsField]:
+    """Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_heading",
+            title="Prowlarr Integration (Optional)",
+            description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.",
+        ),
+    ]
+
+    # Get actual Prowlarr connection fields
+    prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"]
+    for field_key in prowlarr_fields:
+        field = _get_field_from_tab("prowlarr_config", field_key)
+        if field:
+            fields.append(field)
+
+    return fields
+
+
+def get_prowlarr_indexers_fields() -> List[SettingsField]:
+    """Step 5: Select Prowlarr indexers to search."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_indexers_heading",
+            title="Select Indexers",
+            description="Choose which indexers to search for books. Leave empty to search all available indexers.",
+        ),
+    ]
+
+    # Get the indexers multi-select field
+    indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS")
+    if indexers_field:
+        fields.append(indexers_field)
+
+    return fields
+
+
+# =============================================================================
+# Step Configuration
+# =============================================================================
+
+
+ONBOARDING_STEPS = [
+    {
+        "id": "search_mode",
+        "title": "Search Mode",
+        "tab": "search_mode",
+        "get_fields": get_search_mode_fields,
+    },
+    {
+        "id": "metadata_provider",
+        "title": "Metadata Provider",
+        "tab": "search_mode",
+        "get_fields": get_metadata_provider_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+    },
+    {
+        "id": "hardcover_setup",
+        "title": "Hardcover Setup",
+        "tab": "hardcover",
+        "get_fields": get_hardcover_setup_fields,
+        # Must be universal mode AND hardcover selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "hardcover"},
+        ],
+    },
+    {
+        "id": "googlebooks_setup",
+        "title": "Google Books Setup",
+        "tab": "googlebooks",
+        "get_fields": get_googlebooks_setup_fields,
+        # Must be universal mode AND googlebooks selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "googlebooks"},
+        ],
+    },
+    {
+        "id": "prowlarr",
+        "title": "Prowlarr",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+        "optional": True,
+    },
+    {
+        "id": "prowlarr_indexers",
+        "title": "Indexers",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_indexers_fields,
+        # Only show when Prowlarr is enabled
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "PROWLARR_ENABLED", "value": True},
+        ],
+        "optional": True,
+    },
+]
+
+
+def get_onboarding_config() -> Dict[str, Any]:
+    """
+    Get the full onboarding configuration including steps and current values.
+    """
+    steps = []
+    all_values = {}
+
+    for step_config in ONBOARDING_STEPS:
+        fields = step_config["get_fields"]()
+        tab_name = step_config["tab"]
+
+        # Serialize fields with current values
+        serialized_fields = []
+        for field in fields:
+            serialized = serialize_field(field, tab_name, include_value=True)
+            serialized_fields.append(serialized)
+
+            # Collect values (skip HeadingFields)
+            if hasattr(field, 'key') and field.key and not isinstance(field, HeadingField):
+                value = get_setting_value(field, tab_name)
+                all_values[field.key] = value if value is not None else getattr(field, 'default', '')
+
+        step = {
+            "id": step_config["id"],
+            "title": step_config["title"],
+            "tab": tab_name,
+            "fields": serialized_fields,
+        }
+
+        if "show_when" in step_config:
+            step["showWhen"] = step_config["show_when"]
+        if step_config.get("optional"):
+            step["optional"] = True
+
+        steps.append(step)
+
+    return {
+        "steps": steps,
+        "values": all_values,
+        "complete": is_onboarding_complete(),
+    }
+
+
+def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Save onboarding settings and mark as complete.
+
+    Args:
+        values: Dict of field key -> value
+
+    Returns:
+        Dict with success status and message
+    """
+    try:
+        # Group values by their target tab
+        tab_values: Dict[str, Dict[str, Any]] = {}
+
+        for step_config in ONBOARDING_STEPS:
+            tab_name = step_config["tab"]
+            fields = step_config["get_fields"]()
+
+            for field in fields:
+                if isinstance(field, HeadingField):
+                    continue
+
+                key = field.key
+                if key in values:
+                    if tab_name not in tab_values:
+                        tab_values[tab_name] = {}
+                    tab_values[tab_name][key] = values[key]
+
+        # Save each tab's values
+        for tab_name, tab_data in tab_values.items():
+            if tab_data:
+                save_config_file(tab_name, tab_data)
+                logger.info(f"Saved onboarding settings to {tab_name}: {list(tab_data.keys())}")
+
+        # Enable the selected metadata provider
+        search_mode = values.get("SEARCH_MODE", "direct")
+        if search_mode == "universal":
+            provider = values.get("METADATA_PROVIDER", "hardcover")
+            if provider:
+                # Map provider name to its enabled key
+                enabled_key_map = {
+                    "hardcover": "HARDCOVER_ENABLED",
+                    "openlibrary": "OPENLIBRARY_ENABLED",
+                    "googlebooks": "GOOGLEBOOKS_ENABLED",
+                }
+                enabled_key = enabled_key_map.get(provider, f"{provider.upper()}_ENABLED")
+
+                # Get existing provider config and add enabled flag
+                provider_config = {enabled_key: True}
+
+                # Include API key if provided for that provider
+                if provider == "hardcover" and values.get("HARDCOVER_API_KEY"):
+                    provider_config["HARDCOVER_API_KEY"] = values["HARDCOVER_API_KEY"]
+                elif provider == "googlebooks" and values.get("GOOGLEBOOKS_API_KEY"):
+                    provider_config["GOOGLEBOOKS_API_KEY"] = values["GOOGLEBOOKS_API_KEY"]
+
+                save_config_file(provider, provider_config)
+                logger.info(f"Enabled metadata provider: {provider} with keys: {list(provider_config.keys())}")
+
+        # Mark onboarding as complete
+        mark_onboarding_complete()
+
+        # Refresh config
+        try:
+            from shelfmark.core.config import config
+            config.refresh()
+        except ImportError as e:
+            logger.debug(f"Could not refresh config after onboarding: {e}")
+
+        return {"success": True, "message": "Onboarding complete!"}
+
+    except Exception as e:
+        logger.error(f"Failed to save onboarding settings: {e}")
+        return {"success": False, "message": str(e)}

shelfmark/core/path_mappings.py

@@ -0,0 +1,136 @@
+"""Remote path mapping utilities.
+
+Used when an external download client reports a completed download path that does
+not exist inside the Shelfmark runtime environment (commonly different Docker
+volume mounts).
+
+A mapping rewrites a remote path prefix into a local path prefix.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+
+@dataclass(frozen=True)
+class RemotePathMapping:
+    host: str
+    remote_path: str
+    local_path: str
+
+
+def _normalize_prefix(path: str) -> str:
+    normalized = str(path or "").strip()
+    if not normalized:
+        return ""
+
+    normalized = normalized.replace("\\", "/")
+
+    if normalized != "/":
+        normalized = normalized.rstrip("/")
+
+    return normalized
+
+
+def _is_windows_path(path: str) -> bool:
+    """Check if a path looks like a Windows path (has a drive letter like C:/)."""
+    return len(path) >= 2 and path[1] == ":" and path[0].isalpha()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host or "").strip().lower()
+
+
+def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
+    if not value or not isinstance(value, list):
+        return []
+
+    mappings: list[RemotePathMapping] = []
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        host = _normalize_host(row.get("host", ""))
+        remote_path = _normalize_prefix(row.get("remotePath", ""))
+        local_path = _normalize_prefix(row.get("localPath", ""))
+
+        if not host or not remote_path or not local_path:
+            continue
+
+        mappings.append(RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path))
+
+    mappings.sort(key=lambda m: len(m.remote_path), reverse=True)
+    return mappings
+
+
+def remap_remote_to_local_with_match(
+    *,
+    mappings: Iterable[RemotePathMapping],
+    host: str,
+    remote_path: str | Path,
+) -> tuple[Path, bool]:
+    host_normalized = _normalize_host(host)
+    remote_normalized = _normalize_prefix(str(remote_path))
+
+    if not remote_normalized:
+        return Path(str(remote_path)), False
+
+    # Windows paths are case-insensitive, so we need case-insensitive matching
+    # for paths that look like Windows paths (e.g., D:/Torrents)
+    is_windows = _is_windows_path(remote_normalized)
+
+    for mapping in mappings:
+        if _normalize_host(mapping.host) != host_normalized:
+            continue
+
+        remote_prefix = _normalize_prefix(mapping.remote_path)
+        if not remote_prefix:
+            continue
+
+        # For Windows paths, do case-insensitive prefix matching
+        if is_windows:
+            remote_lower = remote_normalized.lower()
+            prefix_lower = remote_prefix.lower()
+            matches = remote_lower == prefix_lower or remote_lower.startswith(prefix_lower + "/")
+        else:
+            matches = remote_normalized == remote_prefix or remote_normalized.startswith(remote_prefix + "/")
+
+        if matches:
+            # Use the length of the original prefix to extract remainder
+            # This preserves the original case in folder names
+            remainder = remote_normalized[len(remote_prefix):]
+            local_prefix = _normalize_prefix(mapping.local_path)
+
+            if remainder.startswith("/"):
+                remainder = remainder[1:]
+
+            remapped = Path(local_prefix) / remainder if remainder else Path(local_prefix)
+            return remapped, True
+
+    return Path(remote_normalized), False
+
+
+def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path) -> Path:
+    remapped, _ = remap_remote_to_local_with_match(
+        mappings=mappings,
+        host=host,
+        remote_path=remote_path,
+    )
+    return remapped
+
+
+def get_client_host_identifier(client: Any) -> Optional[str]:
+    """Return a stable identifier used by the mapping UI.
+
+    Sonarr uses the download client's configured host. Shelfmark currently uses
+    the download client 'name' (e.g. qbittorrent, sabnzbd).
+    """
+
+    name = getattr(client, "name", None)
+    if isinstance(name, str) and name.strip():
+        return name.strip().lower()
+
+    return None

shelfmark/core/prefix_middleware.py

@@ -0,0 +1,31 @@
+"""WSGI middleware for hosting Shelfmark under a URL prefix."""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+
+class PrefixMiddleware:
+    """Strip a configured URL prefix from PATH_INFO before routing."""
+
+    def __init__(self, app, prefix: str, bypass_paths: Optional[Iterable[str]] = None) -> None:
+        self.app = app
+        self.prefix = prefix.rstrip("/")
+        self.bypass_paths = set(bypass_paths or [])
+
+    def __call__(self, environ, start_response):
+        path = environ.get("PATH_INFO", "") or ""
+
+        if path in self.bypass_paths:
+            return self.app(environ, start_response)
+
+        if not self.prefix:
+            return self.app(environ, start_response)
+
+        if path == self.prefix or path.startswith(self.prefix + "/"):
+            environ["SCRIPT_NAME"] = self.prefix
+            environ["PATH_INFO"] = path[len(self.prefix):] or "/"
+            return self.app(environ, start_response)
+
+        start_response("404 Not Found", [("Content-Type", "text/plain")])
+        return [b"Not Found"]