Fix HTTP request behavior (#626)

- Add admin config for self-settings options visibility. Remove delivery
preferences or notifications from the view.
- Add option to use Booklore's Bookdrop API destination instead of a
specific library
- Add download path options for all torrent clients

.github/workflows/build-and-publish-docker-image.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository_owner }}/shelfmark
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -20,15 +20,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - suffix: ""
-            target: cwa-bd
-            image_name_suffix: ""
-          - suffix: "-tor"
-            target: cwa-bd-tor
-            image_name_suffix: "-tor"
-          - suffix: "-extbp"
-            target: cwa-bd-extbp
-            image_name_suffix: "-extbp"
+          - target: shelfmark
+          - target: shelfmark-lite
+            image_name_suffix: "-lite"
     steps:
       - name: Get current date
         id: date
@@ -80,4 +74,71 @@ jobs:
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
+
+  # Create aliases for backwards compatibility
+  create-aliases:
+    needs: build-and-push-images
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    permissions:
+      contents: read
+      packages: write
+    env:
+      # Legacy name for backwards compatibility (hardcoded so it works after rename)
+      LEGACY_NAME: calibre-web-automated-book-downloader
+    steps:
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create legacy aliases
+        run: |
+          # Current image names (follows repo name)
+          STANDARD="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          LITE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-lite"
+
+          # Legacy image names (hardcoded for backwards compatibility)
+          LEGACY="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.LEGACY_NAME }}"
+          LEGACY_TOR="${LEGACY}-tor"
+          LEGACY_EXTBP="${LEGACY}-extbp"
+
+          SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
+
+          # Helper function to create alias with all standard tags
+          create_alias() {
+            local SOURCE=$1
+            local ALIAS=$2
+
+            # Always create SHA tag
+            docker buildx imagetools create -t "${ALIAS}:sha-${SHA_SHORT}" "${SOURCE}:sha-${SHA_SHORT}"
+
+            if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+              VERSION="${{ github.ref_name }}"
+              VERSION_NUM="${VERSION#v}"
+              MINOR="${VERSION_NUM%.*}"
+
+              docker buildx imagetools create -t "${ALIAS}:latest" "${SOURCE}:latest"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION_NUM}" "${SOURCE}:${VERSION_NUM}"
+              docker buildx imagetools create -t "${ALIAS}:${MINOR}" "${SOURCE}:${MINOR}"
+              docker buildx imagetools create -t "${ALIAS}:${VERSION}" "${SOURCE}:${VERSION}"
+            else
+              docker buildx imagetools create -t "${ALIAS}:dev" "${SOURCE}:dev"
+            fi
+          }
+
+          # Create legacy aliases pointing to current images
+          # calibre-web-automated-book-downloader → standard image
+          create_alias "${STANDARD}" "${LEGACY}"
+
+          # calibre-web-automated-book-downloader-tor → standard image
+          create_alias "${STANDARD}" "${LEGACY_TOR}"
+
+          # calibre-web-automated-book-downloader-extbp → lite image
+          create_alias "${LITE}" "${LEGACY_EXTBP}"

.gitignore

@@ -228,3 +228,8 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python
 /downloaded_files
 /.local/
+*.local.*
+AGENTS.md
+.claude/
+.playwright-mcp/
+frontend-dist/

.vscode/launch.json

@@ -2,17 +2,17 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Current CWABD File",
+      "name": "Python Debugger: Current Shelfmark File",
       "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false,
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader",
-        "TEMP_DIR": "/tmp/cwa-book-downloader",
+        "INGEST_DIR": "/tmp/shelfmark",
+        "TEMP_DIR": "/tmp/shelfmark",
         "LOG_LEVEL": "DEBUG",
-        "LOG_ROOT": "/tmp/cwa-book-downloader",
+        "LOG_ROOT": "/tmp/shelfmark",
         "ENABLE_LOGGING": "true",
         "DOCKERMODE": "false",
         "DEBUG": "true",
@@ -27,7 +27,7 @@
       "preLaunchTask": "docker-compose up (dev)", // Spin up dev containers
       "postDebugTask": "docker-compose down (dev)", // Optional: tear them down
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -38,7 +38,7 @@
       "preLaunchTask": "docker-compose up (prod)",
       "postDebugTask": "docker-compose down (prod)",
       "env": {
-        "INGEST_DIR": "/tmp/cwa-book-downloader"
+        "INGEST_DIR": "/tmp/shelfmark"
       },
     },
     {
@@ -54,9 +54,9 @@
   ],
   "compounds": [
     {
-      "name": "Launch CWA-BD",
+      "name": "Launch Shelfmark",
       "configurations": [
-        "Launch cwa-bd app.py",
+        "Launch Shelfmark app.py",
         "Launch Browser"
       ]
     }

Dockerfile

@@ -14,8 +14,9 @@ WORKDIR /frontend
 # Copy frontend package files
 COPY src/frontend/package*.json ./
 
-# Install dependencies
-RUN npm ci
+# Install dependencies (cache mount for faster rebuilds)
+RUN --mount=type=cache,target=/root/.npm \
+    npm ci
 
 # Copy frontend source
 COPY src/frontend/ ./
@@ -44,9 +45,9 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    NAME=Calibre-Web-Automated-Book-Downloader \
+    NAME=Shelfmark \
     PYTHONPATH=/app \
-    # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
+    # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
     LC_ALL=en_US.UTF-8
@@ -67,7 +68,14 @@ RUN apt-get update && \
     # For debug
     zip iputils-ping \
     # For user switching
-    sudo && \
+    sudo \
+    # --- Tor support (activated via USING_TOR=true) ---
+    tor \
+    supervisor \
+    iptables && \
+    # Configure iptables alternatives for tor.sh compatibility
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
     # Cleanup APT cache *after* all installs in this layer
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
@@ -84,12 +92,11 @@ RUN apt-get update && \
 WORKDIR /app
 
 # Install Python dependencies using pip
-# Upgrade pip first, then copy requirements and install
-# Copying requirements-base.txt separately leverages build cache
-COPY requirements-base.txt .
-RUN pip install --no-cache-dir -r requirements-base.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
+# Copying requirements files separately leverages build cache
+# Cache mount persists pip cache between builds for faster installs
+COPY requirements-base.txt requirements-shelfmark.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-base.txt
 
 # Copy application code *after* dependencies are installed
 COPY . .
@@ -100,7 +107,7 @@ COPY --from=frontend-builder /frontend/dist /app/frontend-dist
 # Final setup: permissions and directories in one layer
 # Only creating directories and setting executable bits.
 # Ownership will be handled by the entrypoint script.
-RUN mkdir -p /var/log/cwa-book-downloader /cwa-book-ingest && \
+RUN mkdir -p /var/log/shelfmark /books && \
     chmod +x /app/entrypoint.sh /app/tor.sh /app/genDebug.sh
 
 # Expose the application port
@@ -115,7 +122,7 @@ HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 
-FROM base AS cwa-bd
+FROM base AS shelfmark
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -123,55 +130,34 @@ RUN apt-get update && \
     xvfb \
     # For screen recording
     ffmpeg \
-    # --- Chromium ---
+    # --- Chromium (unpinned - uses latest from Debian repos) ---
+    # Chrome 144+ requires --enable-unsafe-swiftshader for WebGL in Docker.
+    # This flag is set in internal_bypasser.py _get_browser_args()
     chromium \
-    # --- ChromeDriver ---
-    chromium-driver \
+    chromium-common \
     # For tkinter (pyautogui)
     python3-tk \
     # For RAR extraction
     unrar-free && \
     # Create symlink so rarfile library can find unrar
-    ln -sf /usr/bin/unrar-free /usr/bin/unrar
-
-# install additional dependencies
-COPY requirements-cwa-bd.txt ./
-RUN pip install --no-cache-dir -r requirements-cwa-bd.txt && \
-    # Clean root's pip cache
-    rm -rf /root/.cache
-
-# Add this line to grant read/execute permissions to others
-RUN chmod -R o+rx /usr/bin/chromium && \
-    chmod -R o+rx /usr/bin/chromedriver && \
-    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
-
-# Default command to run the application entrypoint script
-CMD ["/app/entrypoint.sh"]
-
-FROM cwa-bd AS cwa-bd-tor
-
-ENV USING_TOR=true
-
-# Install Tor and dependencies
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    # --- Tor ---
-    tor \
-    # --- Supervisor ---
-    supervisor \
-    # --- iptables ---
-    iptables && \
-    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
-    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy && \
-    # Cleanup APT cache *after* all installs in this layer
+    ln -sf /usr/bin/unrar-free /usr/bin/unrar && \
+    # Cleanup APT cache
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Override the default command to run Tor
+# Install additional dependencies (requirements file already copied in base stage)
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-shelfmark.txt
+
+# Grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium && \
+    chmod -R o+rwx /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+
+# Default command to run the application entrypoint script
 CMD ["/app/entrypoint.sh"]
 
-FROM base AS cwa-bd-extbp
+FROM base AS shelfmark-lite
 
 ENV USING_EXTERNAL_BYPASSER=true
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install dev build preview typecheck clean up down docker-build refresh
+.PHONY: help install dev build preview typecheck clean up down docker-build refresh restart
 
 # Frontend directory
 FRONTEND_DIR := src/frontend
@@ -21,6 +21,7 @@ help:
 	@echo "Backend (Docker):"
 	@echo "  up         - Start backend services"
 	@echo "  down       - Stop backend services"
+	@echo "  restart    - Restart backend services (no rebuild)"
 	@echo "  docker-build - Build Docker image"
 	@echo "  refresh    - Rebuild and restart backend services"
 
@@ -70,6 +71,11 @@ docker-build:
 	@echo "Building Docker image..."
 	docker compose -f $(COMPOSE_FILE) build
 
+# Restart backend services (no rebuild)
+restart:
+	@echo "Restarting backend services..."
+	docker compose -f $(COMPOSE_FILE) restart
+
 # Rebuild and restart backend services
 refresh:
 	@echo "Rebuilding and restarting backend services..."

README.v2.md

@@ -1,240 +0,0 @@
-# 📚 Book Downloader
-*calibre-web-automated-book-downloader*
-
-<img src="src/frontend/public/logo.png" alt="Book Downloader" width="200">
-
-A unified web interface for searching and downloading books from multiple sources — all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library.
-
-**Fully standalone** — no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) or [Booklore](https://github.com/booklore-app/booklore) for automatic import.
-
-## ✨ Features
-
-- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
-- **Real-Time Progress** - Unified download queue with live status updates across all sources
-- **Two Search Modes**:
-  - **Direct Download** - Search and download from popular web sources
-  - **Universal Mode** - Search metadata providers (Hardcover, Open Library) for richer book discovery and multi-source downloads *(additional sources in development - coming soon!)*
-- **Format Support** - EPUB, MOBI, AZW3, FB2, DJVU, CBZ, CBR and more
-- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
-- **PWA Support** - Install as a mobile app for quick access
-- **Docker Deployment** - Up and running in minutes
-
-## 🖼️ Screenshots
-
-**Home screen**
-![Home screen](README_images/homescreen.png 'Home screen')
-
-**Search results**
-![Search results](README_images/search-results.png 'Search results')
-
-**Multi-source downloads**
-![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
-
-**Download queue**
-![Download queue](README_images/downloads.png 'Download queue')
-
-## 🚀 Quick Start
-
-### Prerequisites
-
-- Docker & Docker Compose
-
-### Installation
-
-1. Download the docker-compose file:
-   ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.yml
-   ```
-
-2. Start the service:
-   ```bash
-   docker compose up -d
-   ```
-
-3. Open `http://localhost:8084`
-
-That's it! Configure settings through the web interface as needed.
-
-### Volume Setup
-
-```yaml
-volumes:
-  - /your/config/path:/config             # Config, database, and artwork cache directory
-  - /your/download/path:/cwa-book-ingest  # Downloaded books
-```
-
-> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
-
-> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
-
-## ⚙️ Configuration
-
-### Search Modes
-
-**Direct Download Mode** (default)
-- Works out of the box, no setup required
-- Searches a huge library of books directly
-- Returns downloadable releases immediately
-
-**Universal Mode**
-- Cleaner search results via metadata providers (Hardcover, Open Library)
-- Aggregates releases from multiple configured sources
-- Requires manual setup (API keys, additional sources)
-
-Set the mode via Settings or `SEARCH_MODE` environment variable.
-
-### Environment Variables
-
-Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
-
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `FLASK_PORT` | Web interface port | `8084` |
-| `INGEST_DIR` | Book download directory | `/cwa-book-ingest` |
-| `TZ` | Container timezone | `UTC` |
-| `UID` / `GID` | Runtime user/group ID | `1000` / `100` |
-| `SEARCH_MODE` | `direct` or `universal` | `direct` |
-
-Some of the additional options available in Settings:
-- **AA Donator Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
-- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
-- **Content Folders** - Route fiction, non-fiction, comics, etc. to separate directories
-- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable
-- **Format & Language** - Filter downloads by preferred formats and languages
-- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
-
-## 🐳 Docker Variants
-
-### Standard
-```bash
-docker compose up -d
-```
-
-### Tor Variant
-Routes all traffic through Tor for enhanced privacy:
-```bash
-curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.tor.yml
-docker compose -f docker-compose.tor.yml up -d
-```
-
-**Notes:**
-- Requires `NET_ADMIN` and `NET_RAW` capabilities
-- Timezone is auto-detected from Tor exit node
-- Custom DNS/proxy settings are ignored
-
-### External Cloudflare Resolver
-Use FlareSolverr or ByParr instead of the built-in bypasser:
-```bash
-curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.extbp.yml
-docker compose -f docker-compose.extbp.yml up -d
-```
-
-Configure the resolver URL in Settings under the Cloudflare tab.
-
-**When to use external vs internal bypasser:**
-- **External** is useful if you already run FlareSolverr for other services (saves resources) or if you rarely need bypassing
-- **Internal** (default) is faster and more reliable for most users - it's optimized specifically for this application
-
-## 🔐 Authentication
-
-Authentication is optional but recommended for shared or exposed instances. Enable in Settings.
-
-**Alternative**: If you're running Calibre-Web, you can reuse its user database by mounting it:
-
-```yaml
-volumes:
-  - /path/to/calibre-web/app.db:/auth/app.db:ro
-```
-
-## Health Monitoring
-
-The application exposes a health endpoint at `/api/status`. Add a health check to your compose:
-
-```yaml
-healthcheck:
-  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/status"]
-  interval: 30s
-  timeout: 30s
-  retries: 3
-```
-
-## Logging
-
-Logs are available via:
-- `docker logs <container-name>`
-- `/var/log/cwa-book-downloader/` inside the container (when `ENABLE_LOGGING=true`)
-
-Log level is configurable via Settings or `LOG_LEVEL` environment variable.
-
-## Development
-
-```bash
-# Frontend development
-make install     # Install dependencies
-make dev         # Start Vite dev server (localhost:5173)
-make build       # Production build
-make typecheck   # TypeScript checks
-
-# Backend (Docker)
-make up          # Start backend via docker-compose.dev.yml
-make down        # Stop services
-make refresh     # Rebuild and restart
-```
-
-The frontend dev server proxies to the backend on port 8084.
-
-### Architecture
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│                      Web Interface                          │
-│                 (React + TypeScript + Vite)                 │
-├─────────────────────────────────────────────────────────────┤
-│                      Flask Backend                          │
-│                   (REST API + WebSocket)                    │
-├───────────────────┬─────────────────────┬───────────────────┤
-│ Metadata Providers│   Download Queue    │  Cloudflare       │
-│                   │   & Orchestrator    │  Bypass           │
-├───────────────────┼─────────────────────┼───────────────────┤
-│ • Hardcover       │ • Task scheduling   │ • Internal        │
-│ • Open Library    │ • Progress tracking │ • External        │
-│                   │ • Retry logic       │   (FlareSolverr)  │
-├───────────────────┴─────────────────────┴───────────────────┤
-│                     Release Sources                         │
-├─────────────────────────────────────────────────────────────┤
-│ • Direct Download (Anna's Archive → Libgen → Welib)         │
-├─────────────────────────────────────────────────────────────┤
-│                     Network Layer                           │
-├─────────────────────────────────────────────────────────────┤
-│ • Auto DNS rotation  • Mirror failover  • Resume support    │
-└─────────────────────────────────────────────────────────────┘
-```
-
-The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
-
-## Contributing
-
-Contributions are welcome! Please file issues or submit pull requests on GitHub.
-
-> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR!
-
-## License
-
-MIT License - see [LICENSE](LICENSE) for details.
-
-## ⚠️ Disclaimers
-
-### Copyright Notice
-
-This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
-- Ensuring they have the right to download requested materials
-- Respecting copyright laws and intellectual property rights
-- Using the tool in compliance with their local regulations
-
-### Library Integration
-
-Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
-
-## Support
-
-For issues or questions, please [file an issue](https://github.com/calibrain/calibre-web-automated-book-downloader/issues) on GitHub.

compose/docker-compose.lite.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark-lite:
+    image: ghcr.io/calibrain/shelfmark-lite:latest
+    environment:
+      # EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.tor.yml

@@ -0,0 +1,20 @@
+# Routes all traffic through Tor - requires NET_ADMIN capability
+services:
+  shelfmark-tor:
+    image: ghcr.io/calibrain/shelfmark:latest
+    environment:
+      FLASK_PORT: 8084
+      USING_TOR: true
+      PUID: 1000
+      PGID: 1000
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

compose/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    container_name: shelfmark
+    environment:
+      PUID: 1000
+      PGID: 1000
+    ports:
+      - 8084:8084
+    restart: unless-stopped
+    volumes:
+      - /path/to/books:/books # Default destination for book downloads
+      - /path/to/config:/config # App configuration
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

cwa_book_downloader/__init__.py

@@ -1 +0,0 @@
-"""CWA Book Downloader - book search and download service."""

cwa_book_downloader/__main__.py

@@ -1,8 +0,0 @@
-"""Package entry point for `python -m cwa_book_downloader`."""
-
-from cwa_book_downloader.main import app, socketio
-from cwa_book_downloader.config.env import FLASK_HOST, FLASK_PORT
-from cwa_book_downloader.core.config import config
-
-if __name__ == "__main__":
-    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

cwa_book_downloader/bypass/__init__.py

@@ -1 +0,0 @@
-"""Cloudflare bypass utilities."""

cwa_book_downloader/bypass/external_bypasser.py

@@ -1,162 +0,0 @@
-"""External Cloudflare bypasser using FlareSolverr."""
-
-from threading import Event
-from typing import Optional, TYPE_CHECKING
-import requests
-import time
-import random
-
-from cwa_book_downloader.core.config import config
-from cwa_book_downloader.core.logger import setup_logger
-
-if TYPE_CHECKING:
-    from cwa_book_downloader.download import network
-
-
-class BypassCancelledException(Exception):
-    """Raised when a bypass operation is cancelled."""
-    pass
-
-logger = setup_logger(__name__)
-
-# Connection timeout (seconds) - how long to wait for external bypasser to accept connection
-CONNECT_TIMEOUT = 10
-# Maximum read timeout cap (seconds) - hard limit regardless of EXT_BYPASSER_TIMEOUT
-MAX_READ_TIMEOUT = 120
-# Buffer added to bypasser's configured timeout (seconds) - accounts for processing overhead
-READ_TIMEOUT_BUFFER = 15
-# Retry settings for bypasser failures
-MAX_RETRY = 5
-BACKOFF_BASE = 1.0
-BACKOFF_CAP = 10.0
-
-
-def _fetch_via_bypasser(target_url: str) -> Optional[str]:
-    """Make a single request to the external bypasser service.
-
-    Args:
-        target_url: The URL to fetch through the bypasser
-
-    Returns:
-        HTML content if successful, None otherwise
-    """
-    bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
-    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
-    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
-
-    if not bypasser_url or not bypasser_path:
-        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
-        return None
-
-    bypasser_endpoint = f"{bypasser_url}{bypasser_path}"
-    headers = {"Content-Type": "application/json"}
-    payload = {
-        "cmd": "request.get",
-        "url": target_url,
-        "maxTimeout": bypasser_timeout
-    }
-
-    # Calculate read timeout: bypasser timeout (ms -> s) + buffer, capped at max
-    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
-
-    try:
-        response = requests.post(
-            bypasser_endpoint,
-            headers=headers,
-            json=payload,
-            timeout=(CONNECT_TIMEOUT, read_timeout)
-        )
-        response.raise_for_status()
-        result = response.json()
-
-        status = result.get('status', 'unknown')
-        message = result.get('message', '')
-        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
-
-        # Check for error status (bypasser returns status="error" with solution=null on failure)
-        if status != 'ok':
-            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
-            return None
-
-        solution = result.get('solution')
-        if not solution:
-            logger.warning(f"External bypasser returned empty solution for '{target_url}'")
-            return None
-
-        html = solution.get('response', '')
-        if not html:
-            logger.warning(f"External bypasser returned empty response for '{target_url}'")
-            return None
-
-        return html
-
-    except requests.exceptions.Timeout:
-        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
-        return None
-    except requests.exceptions.RequestException as e:
-        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
-        return None
-    except (KeyError, TypeError, ValueError) as e:
-        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
-        return None
-
-
-def get_bypassed_page(url: str, selector: Optional["network.AAMirrorSelector"] = None, cancel_flag: Optional[Event] = None) -> Optional[str]:
-    """Fetch HTML content from a URL using an external Cloudflare bypasser service.
-
-    Retries with exponential backoff and mirror/DNS rotation on failure.
-
-    Args:
-        url: Target URL to fetch
-        selector: Mirror selector for AA URL rewriting and rotation
-        cancel_flag: Optional threading Event to signal cancellation
-
-    Returns:
-        HTML content if successful, None otherwise
-
-    Raises:
-        BypassCancelledException: If cancel_flag is set during operation
-    """
-    from cwa_book_downloader.download import network as network_module
-    sel = selector or network_module.AAMirrorSelector()
-
-    for attempt in range(1, MAX_RETRY + 1):
-        # Check for cancellation before each attempt
-        if cancel_flag and cancel_flag.is_set():
-            logger.info("External bypasser cancelled by user")
-            raise BypassCancelledException("Bypass cancelled")
-
-        attempt_url = sel.rewrite(url)
-        result = _fetch_via_bypasser(attempt_url)
-        if result:
-            return result
-
-        if attempt == MAX_RETRY:
-            break
-
-        # Check for cancellation before backoff wait
-        if cancel_flag and cancel_flag.is_set():
-            logger.info("External bypasser cancelled during retry")
-            raise BypassCancelledException("Bypass cancelled")
-
-        # Backoff with jitter before retry, checking cancellation during wait
-        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
-        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
-
-        # Check cancellation during delay (check every second)
-        for _ in range(int(delay)):
-            if cancel_flag and cancel_flag.is_set():
-                logger.info("External bypasser cancelled during backoff")
-                raise BypassCancelledException("Bypass cancelled")
-            time.sleep(1)
-        # Sleep remaining fraction
-        remaining = delay - int(delay)
-        if remaining > 0:
-            time.sleep(remaining)
-
-        # Rotate mirror/DNS for next attempt
-        new_base, action = sel.next_mirror_or_rotate_dns()
-        if action in ("mirror", "dns") and new_base:
-            logger.info(f"Rotated {action} for retry")
-
-    return None

cwa_book_downloader/config/env.py

@@ -1,151 +0,0 @@
-"""Environment variable parsing. No local dependencies - import first."""
-
-import os
-import shutil
-from pathlib import Path
-
-
-def string_to_bool(s: str) -> bool:
-    return s.lower() in ["true", "yes", "1", "y"]
-
-
-# Authentication and session settings
-SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
-
-CWA_DB = os.getenv("CWA_DB_PATH")
-CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
-CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
-LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
-LOG_DIR = LOG_ROOT / "cwa-book-downloader"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
-INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
-INGEST_DIR_BOOK_FICTION = os.getenv("INGEST_DIR_BOOK_FICTION", "")
-INGEST_DIR_BOOK_NON_FICTION = os.getenv("INGEST_DIR_BOOK_NON_FICTION", "")
-INGEST_DIR_BOOK_UNKNOWN = os.getenv("INGEST_DIR_BOOK_UNKNOWN", "")
-INGEST_DIR_MAGAZINE = os.getenv("INGEST_DIR_MAGAZINE", "")
-INGEST_DIR_COMIC_BOOK = os.getenv("INGEST_DIR_COMIC_BOOK", "")
-INGEST_DIR_AUDIOBOOK = os.getenv("INGEST_DIR_AUDIOBOOK", "")
-INGEST_DIR_STANDARDS_DOCUMENT = os.getenv("INGEST_DIR_STANDARDS_DOCUMENT", "")
-INGEST_DIR_MUSICAL_SCORE = os.getenv("INGEST_DIR_MUSICAL_SCORE", "")
-INGEST_DIR_OTHER = os.getenv("INGEST_DIR_OTHER", "")
-DOWNLOAD_PATHS = {
-    "book (fiction)": Path(INGEST_DIR_BOOK_FICTION) if INGEST_DIR_BOOK_FICTION else INGEST_DIR,
-    "book (non-fiction)": Path(INGEST_DIR_BOOK_NON_FICTION) if INGEST_DIR_BOOK_NON_FICTION else INGEST_DIR,
-    "book (unknown)": Path(INGEST_DIR_BOOK_UNKNOWN) if INGEST_DIR_BOOK_UNKNOWN else INGEST_DIR,
-    "magazine": Path(INGEST_DIR_MAGAZINE) if INGEST_DIR_MAGAZINE else INGEST_DIR,
-    "comic book": Path(INGEST_DIR_COMIC_BOOK) if INGEST_DIR_COMIC_BOOK else INGEST_DIR,
-    "audiobook": Path(INGEST_DIR_AUDIOBOOK) if INGEST_DIR_AUDIOBOOK else INGEST_DIR,
-    "standards document": Path(INGEST_DIR_STANDARDS_DOCUMENT) if INGEST_DIR_STANDARDS_DOCUMENT else INGEST_DIR,
-    "musical score": Path(INGEST_DIR_MUSICAL_SCORE) if INGEST_DIR_MUSICAL_SCORE else INGEST_DIR,
-    "other": Path(INGEST_DIR_OTHER) if INGEST_DIR_OTHER else INGEST_DIR,
-}
-
-STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
-USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
-MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
-DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
-USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
-HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
-HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
-AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
-_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
-_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
-_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
-_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
-_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
-FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
-FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
-# Debug: skip specific download sources for testing fallback chains
-# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
-_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
-DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
-
-# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
-# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
-_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
-_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
-
-# Version information from Docker build
-BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
-RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
-
-# If debug is true, we want to log everything
-if DEBUG:
-    LOG_LEVEL = "DEBUG"
-else:
-    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
-MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
-MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1"))
-DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip()
-USE_DOH = string_to_bool(os.getenv("USE_DOH", "true"))
-BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
-BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true"))
-
-# Logging settings
-LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
-
-USING_EXTERNAL_BYPASSER = string_to_bool(os.getenv("USING_EXTERNAL_BYPASSER", "false"))
-if USING_EXTERNAL_BYPASSER:
-    EXT_BYPASSER_URL = os.getenv("EXT_BYPASSER_URL", "http://flaresolverr:8191").strip()
-    EXT_BYPASSER_PATH = os.getenv("EXT_BYPASSER_PATH", "/v1").strip()
-    EXT_BYPASSER_TIMEOUT = int(os.getenv("EXT_BYPASSER_TIMEOUT", "60000"))
-
-USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
-# If using Tor, we don't need to set custom DNS, use DOH, or proxy
-if USING_TOR:
-    _CUSTOM_DNS = ""
-    USE_DOH = False
-    HTTP_PROXY = ""
-    HTTPS_PROXY = ""
-
-# Detect Tor variant (has tor binary installed)
-TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
-
-# Calibre-Web URL for navigation button
-CALIBRE_WEB_URL = os.getenv("CALIBRE_WEB_URL", "").strip()
-
-# Metadata provider settings (Stage 2)
-# Set to "hardcover" or "openlibrary" to enable metadata-first search mode
-METADATA_PROVIDER = os.getenv("METADATA_PROVIDER", "").strip().lower()
-HARDCOVER_API_KEY = os.getenv("HARDCOVER_API_KEY", "").strip()
-
-# Cache TTL settings (in seconds)
-METADATA_CACHE_SEARCH_TTL = int(os.getenv("METADATA_CACHE_SEARCH_TTL", "300"))  # 5 minutes
-METADATA_CACHE_BOOK_TTL = int(os.getenv("METADATA_CACHE_BOOK_TTL", "600"))  # 10 minutes
-
-# Cover image cache settings
-def _is_config_dir_writable() -> bool:
-    """Check if the config directory exists and is writable."""
-    try:
-        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
-            return False
-        test_file = CONFIG_DIR / ".write_test"
-        test_file.touch()
-        test_file.unlink()
-        return True
-    except (OSError, PermissionError):
-        return False
-
-
-def is_covers_cache_enabled() -> bool:
-    """Check if cover caching is enabled (dynamic, respects settings changes).
-
-    Cache is only enabled if:
-    1. The COVERS_CACHE_ENABLED setting is true
-    2. The config directory is writable
-    """
-    from cwa_book_downloader.core.config import config
-    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
-    return setting_enabled and _is_config_dir_writable()
-
-
-# Legacy static value - use is_covers_cache_enabled() for dynamic checks
-_COVERS_CACHE_ENABLED_ENV = string_to_bool(os.getenv("COVERS_CACHE_ENABLED", "true"))
-COVERS_CACHE_ENABLED = _COVERS_CACHE_ENABLED_ENV and _is_config_dir_writable()
-COVERS_CACHE_DIR = CONFIG_DIR / "covers"
-COVERS_CACHE_TTL = int(os.getenv("COVERS_CACHE_TTL", "0"))  # 0 = forever (covers are static)
-COVERS_CACHE_MAX_SIZE_MB = int(os.getenv("COVERS_CACHE_MAX_SIZE_MB", "500"))

cwa_book_downloader/config/security.py

@@ -1,152 +0,0 @@
-"""Authentication settings registration."""
-
-from typing import Any, Dict
-
-from werkzeug.security import generate_password_hash
-
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.settings_registry import (
-    register_settings,
-    register_on_save,
-    load_config_file,
-    TextField,
-    PasswordField,
-    CheckboxField,
-    ActionButton,
-)
-
-logger = setup_logger(__name__)
-
-
-def _clear_builtin_credentials() -> Dict[str, Any]:
-    """Clear built-in credentials to allow public access."""
-    try:
-        config = load_config_file("security")
-        config.pop("BUILTIN_USERNAME", None)
-        config.pop("BUILTIN_PASSWORD_HASH", None)
-
-        # Save the cleared config
-        from cwa_book_downloader.core.settings_registry import _get_config_file_path, _ensure_config_dir
-        import json
-
-        _ensure_config_dir("security")
-        config_path = _get_config_file_path("security")
-        with open(config_path, 'w') as f:
-            json.dump(config, f, indent=2)
-
-        logger.info("Cleared credentials")
-        return {"success": True, "message": "Credentials cleared. The app is now publicly accessible."}
-
-    except Exception as e:
-        logger.error(f"Failed to clear credentials: {e}")
-        return {"success": False, "message": f"Failed to clear credentials: {str(e)}"}
-
-
-def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Custom save handler for security settings.
-
-    Handles password validation and hashing:
-    - If new password is provided, validate confirmation and hash it
-    - If password fields are empty, preserve existing hash
-    - Never store raw passwords
-
-    Returns:
-        Dict with processed values to save and any validation errors.
-    """
-    password = values.get("BUILTIN_PASSWORD", "")
-    password_confirm = values.get("BUILTIN_PASSWORD_CONFIRM", "")
-
-    # Remove raw password fields - they should never be persisted
-    values.pop("BUILTIN_PASSWORD", None)
-    values.pop("BUILTIN_PASSWORD_CONFIRM", None)
-
-    # If password is provided, validate and hash it
-    if password:
-        if password != password_confirm:
-            return {
-                "error": True,
-                "message": "Passwords do not match",
-                "values": values
-            }
-
-        if len(password) < 4:
-            return {
-                "error": True,
-                "message": "Password must be at least 4 characters",
-                "values": values
-            }
-
-        # Hash the password
-        values["BUILTIN_PASSWORD_HASH"] = generate_password_hash(password)
-        logger.info("Password hash updated")
-
-    # If no password provided but username is being set, preserve existing hash
-    elif "BUILTIN_USERNAME" in values:
-        existing = load_config_file("security")
-        if "BUILTIN_PASSWORD_HASH" in existing:
-            values["BUILTIN_PASSWORD_HASH"] = existing["BUILTIN_PASSWORD_HASH"]
-
-    return {"error": False, "values": values}
-
-
-@register_settings("security", "Security", icon="shield", order=5)
-def security_settings():
-    """Security and authentication settings."""
-    from cwa_book_downloader.config.env import CWA_DB_PATH
-    import os
-
-    cwa_db_available = CWA_DB_PATH and os.path.exists(CWA_DB_PATH)
-
-    fields = [
-        TextField(
-            key="BUILTIN_USERNAME",
-            label="Username",
-            description="Set a username and password to require login. Leave both empty for public access.",
-            placeholder="Enter username",
-            env_supported=False,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        PasswordField(
-            key="BUILTIN_PASSWORD",
-            label="Set Password",
-            description="Fill in to set or change the password.",
-            placeholder="Enter new password",
-            env_supported=False,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        PasswordField(
-            key="BUILTIN_PASSWORD_CONFIRM",
-            label="Confirm Password",
-            placeholder="Confirm new password",
-            env_supported=False,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        ActionButton(
-            key="clear_credentials",
-            label="Clear Credentials",
-            description="Remove login requirement and make the app publicly accessible.",
-            style="danger",
-            callback=_clear_builtin_credentials,
-            disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
-        ),
-        CheckboxField(
-            key="USE_CWA_AUTH",
-            label="Use Calibre-Web Database",
-            description=(
-                "Authenticate using your existing Calibre-Web users instead of the credentials above."
-                if cwa_db_available
-                else "Authenticate using your existing Calibre-Web users. Set the CWA_DB_PATH environment variable to your Calibre-Web app.db file to enable this option."
-            ),
-            default=False,
-            env_supported=False,
-            disabled=not cwa_db_available,
-            disabled_reason="Set the CWA_DB_PATH environment variable to your Calibre-Web app.db file path to enable this option.",
-        ),
-    ]
-
-    return fields
-
-
-# Register the on_save handler for this tab
-register_on_save("security", _on_save_security)

cwa_book_downloader/config/settings.py

@@ -1,871 +0,0 @@
-"""Core settings registration and derived configuration values."""
-
-import os
-from pathlib import Path
-import json
-
-from cwa_book_downloader.config import env
-from cwa_book_downloader.core.logger import setup_logger
-
-logger = setup_logger(__name__)
-
-# Log configuration values at DEBUG level, filtering out module imports and functions
-logger.debug("Environment configuration:")
-for key, value in env.__dict__.items():
-    # Skip private attributes, modules, types, and callables (functions)
-    if key.startswith('_'):
-        continue
-    if isinstance(value, type) or callable(value):
-        continue
-    # Don't log module objects (they have __name__ attribute)
-    if hasattr(value, '__name__') and hasattr(value, '__file__'):
-        continue
-    # Redact sensitive values
-    if key == "AA_DONATOR_KEY" and isinstance(value, str) and value.strip():
-        value = "REDACTED"
-    if key == "HARDCOVER_API_KEY" and isinstance(value, str) and value.strip():
-        value = "REDACTED"
-    logger.debug(f"  {key}: {value}")
-
-# Load supported book languages from data file
-# Path is relative to the package root, not this file
-_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
-with open(_DATA_DIR / "book-languages.json") as file:
-    _SUPPORTED_BOOK_LANGUAGE = json.load(file)
-
-# Directory settings
-BASE_DIR = Path(__file__).resolve().parent.parent.parent
-logger.debug(f"BASE_DIR: {BASE_DIR}")
-if env.ENABLE_LOGGING:
-    env.LOG_DIR.mkdir(exist_ok=True)
-
-# Create necessary directories
-env.TMP_DIR.mkdir(exist_ok=True)
-env.INGEST_DIR.mkdir(exist_ok=True)
-
-CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
-logger.debug(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
-logger.debug(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
-logger.debug(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
-
-# DNS placeholders - actual values set by network.init() from config/ENV
-CUSTOM_DNS: list[str] = []
-DOH_SERVER: str = ""
-
-# Warn about external bypasser DNS limitations
-if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS:
-    logger.warning(
-        "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
-        "not this application's custom DNS settings. If you experience DNS-related blocks, "
-        "configure DNS at the Docker/system level for your FlareSolverr container, "
-        "or consider using the internal bypasser which integrates with the app's DNS system."
-    )
-
-# Proxy settings
-PROXIES = {}
-if env.HTTP_PROXY:
-    PROXIES["http"] = env.HTTP_PROXY
-if env.HTTPS_PROXY:
-    PROXIES["https"] = env.HTTPS_PROXY
-logger.debug(f"PROXIES: {PROXIES}")
-
-# Anna's Archive settings
-AA_BASE_URL = env._AA_BASE_URL
-AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
-AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
-AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
-
-# File format settings
-SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
-logger.debug(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
-
-# Complex language processing logic kept in config.py
-BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
-BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
-if len(BOOK_LANGUAGE) == 0:
-    BOOK_LANGUAGE = ['en']
-
-# Custom script settings with validation logic
-CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
-if CUSTOM_SCRIPT:
-    if not os.path.exists(CUSTOM_SCRIPT):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
-        CUSTOM_SCRIPT = ""
-    elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
-        CUSTOM_SCRIPT = ""
-
-# Debugging settings
-if not env.USING_EXTERNAL_BYPASSER:
-    # Virtual display settings for debugging internal cloudflare bypasser
-    VIRTUAL_SCREEN_SIZE = (1024, 768)
-    RECORDING_DIR = env.LOG_DIR / "recording"
-
-
-from cwa_book_downloader.core.settings_registry import (
-    register_settings,
-    register_group,
-    TextField,
-    PasswordField,
-    NumberField,
-    CheckboxField,
-    SelectField,
-    MultiSelectField,
-    OrderableListField,
-    HeadingField,
-    ActionButton,
-)
-
-
-register_group(
-    "direct_download",
-    "Anna's Archive",
-    icon="download",
-    order=20
-)
-
-register_group(
-    "metadata_providers",
-    "Metadata Providers",
-    icon="book",
-    order=12  # Between Network (10) and Advanced (15)
-)
-
-
-# Anna's Archive sort options (for Direct mode)
-_AA_SORT_OPTIONS = [
-    {"value": "relevance", "label": "Most relevant"},
-    {"value": "newest", "label": "Newest (publication year)"},
-    {"value": "oldest", "label": "Oldest (publication year)"},
-    {"value": "largest", "label": "Largest (filesize)"},
-    {"value": "smallest", "label": "Smallest (filesize)"},
-    {"value": "newest_added", "label": "Newest (open sourced)"},
-    {"value": "oldest_added", "label": "Oldest (open sourced)"},
-]
-
-_FORMAT_OPTIONS = [
-    {"value": "epub", "label": "EPUB"},
-    {"value": "mobi", "label": "MOBI"},
-    {"value": "azw3", "label": "AZW3"},
-    {"value": "pdf", "label": "PDF"},
-    {"value": "fb2", "label": "FB2"},
-    {"value": "djvu", "label": "DJVU"},
-    {"value": "cbz", "label": "CBZ"},
-    {"value": "cbr", "label": "CBR"},
-    {"value": "txt", "label": "TXT"},
-    {"value": "rtf", "label": "RTF"},
-    {"value": "doc", "label": "DOC"},
-    {"value": "docx", "label": "DOCX"},
-    {"value": "zip", "label": "ZIP"},
-    {"value": "rar", "label": "RAR"},
-]
-
-
-def _get_metadata_provider_options():
-    """Build metadata provider options dynamically from enabled providers only."""
-    from cwa_book_downloader.metadata_providers import list_providers, is_provider_enabled
-
-    options = []
-    for provider in list_providers():
-        # Only show providers that are enabled
-        if is_provider_enabled(provider["name"]):
-            options.append({"value": provider["name"], "label": provider["display_name"]})
-
-    # If no providers enabled, show a placeholder option
-    if not options:
-        options = [
-            {"value": "", "label": "No providers enabled"},
-        ]
-
-    return options
-
-
-def _get_release_source_options():
-    """Build release source options dynamically from registered sources."""
-    from cwa_book_downloader.release_sources import list_available_sources
-
-    return [
-        {"value": source["name"], "label": source["display_name"]}
-        for source in list_available_sources()
-    ]
-
-_LANGUAGE_OPTIONS = [{"value": lang["code"], "label": lang["language"]} for lang in _SUPPORTED_BOOK_LANGUAGE]
-
-
-def _clear_covers_cache(current_values: dict) -> dict:
-    """Clear the cover image cache."""
-    try:
-        from cwa_book_downloader.core.image_cache import get_image_cache, reset_image_cache
-
-        cache = get_image_cache()
-        count = cache.clear()
-
-        # Reset the singleton so it reinitializes with fresh state
-        reset_image_cache()
-
-        return {
-            "success": True,
-            "message": f"Cleared {count} cached cover images.",
-        }
-    except Exception as e:
-        logger.error(f"Failed to clear cover cache: {e}")
-        return {
-            "success": False,
-            "message": f"Failed to clear cache: {str(e)}",
-        }
-
-
-def _clear_metadata_cache(current_values: dict) -> dict:
-    """Clear the in-memory metadata cache."""
-    try:
-        from cwa_book_downloader.core.cache import get_metadata_cache
-
-        cache = get_metadata_cache()
-        stats_before = cache.stats()
-        cache.clear()
-
-        return {
-            "success": True,
-            "message": f"Cleared {stats_before['size']} cached entries.",
-        }
-    except Exception as e:
-        logger.error(f"Failed to clear metadata cache: {e}")
-        return {
-            "success": False,
-            "message": f"Failed to clear cache: {str(e)}",
-        }
-
-
-@register_settings("general", "General", icon="settings", order=0)
-def general_settings():
-    """Core application settings."""
-    return [
-        TextField(
-            key="CALIBRE_WEB_URL",
-            label="Book Management App URL",
-            description="Adds a navigation button to your book manager instance (Calibre-Web Automated, Booklore, etc).",
-            placeholder="http://calibre-web:8083",
-        ),
-        HeadingField(
-            key="search_mode_heading",
-            title="Search Mode",
-            description="Direct searches Anna's Archive and downloads immediately. Universal searches book metadata first, letting you choose from multiple release sources including Anna's Archive and Prowlarr.",
-        ),
-        SelectField(
-            key="SEARCH_MODE",
-            label="Search Mode",
-            description="How you want to search for and download books.",
-            options=[
-                {
-                    "value": "direct",
-                    "label": "Direct (Anna's Archive)",
-                    "description": "Search Anna's Archive and download directly. Works out of the box.",
-                },
-                {
-                    "value": "universal",
-                    "label": "Universal",
-                    "description": "Metadata-based search with downloads from all sources.",
-                },
-            ],
-            default="direct",
-        ),
-        SelectField(
-            key="AA_DEFAULT_SORT",
-            label="Default Sort Order",
-            description="Default sort order for Anna's Archive search results.",
-            options=_AA_SORT_OPTIONS,
-            default="relevance",
-            env_supported=False,  # UI-only setting
-            show_when={"field": "SEARCH_MODE", "value": "direct"},
-        ),
-        SelectField(
-            key="METADATA_PROVIDER",
-            label="Metadata Provider",
-            description="Choose which metadata provider to use for book searches.",
-            options=_get_metadata_provider_options,  # Callable - evaluated lazily to avoid circular imports
-            default="openlibrary",
-            show_when={"field": "SEARCH_MODE", "value": "universal"},
-        ),
-        SelectField(
-            key="DEFAULT_RELEASE_SOURCE",
-            label="Default Release Source",
-            description="The release source tab to open by default in the release modal.",
-            options=_get_release_source_options,  # Callable - evaluated lazily to avoid circular imports
-            default="direct_download",
-            env_supported=False,  # UI-only setting, not configurable via ENV
-            show_when={"field": "SEARCH_MODE", "value": "universal"},
-        ),
-        HeadingField(
-            key="search_defaults_heading",
-            title="Default Search Options",
-            description="Default filters applied to searches. Can be overridden using advanced search options.",
-        ),
-        MultiSelectField(
-            key="SUPPORTED_FORMATS",
-            label="Supported Formats",
-            description="Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found.",
-            options=_FORMAT_OPTIONS,
-            default=["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
-        ),
-        MultiSelectField(
-            key="BOOK_LANGUAGE",
-            label="Default Book Languages",
-            description="Default language filter for searches.",
-            options=_LANGUAGE_OPTIONS,
-            default=["en"],
-        ),
-    ]
-
-
-@register_settings("network", "Network", icon="globe", order=10)
-def network_settings():
-    """Network and connectivity settings."""
-    # Check if Tor variant is available and if Tor is currently enabled
-    tor_available = env.TOR_VARIANT_AVAILABLE
-    tor_enabled = env.USING_TOR
-
-    # When Tor is enabled (only possible in Tor variant), DNS/proxy settings are overridden
-    # The Tor variant uses iptables to force ALL traffic through Tor - it cannot be disabled
-    tor_overrides_network = tor_available  # If Tor variant, network settings are always managed by Tor
-
-    return [
-        SelectField(
-            key="CUSTOM_DNS",
-            label="DNS Provider",
-            description=(
-                "Managed by Tor when Tor routing is enabled."
-                if tor_overrides_network
-                else "DNS provider for domain resolution. 'Auto' rotates through providers on failure."
-            ),
-            options=[
-                {"value": "auto", "label": "Auto (Recommended)"},
-                {"value": "system", "label": "System"},
-                {"value": "google", "label": "Google"},
-                {"value": "cloudflare", "label": "Cloudflare"},
-                {"value": "quad9", "label": "Quad9"},
-                {"value": "opendns", "label": "OpenDNS"},
-                {"value": "manual", "label": "Manual"},
-            ],
-            default="auto",
-            disabled=tor_overrides_network,
-            disabled_reason="DNS is managed by Tor when Tor routing is enabled.",
-        ),
-        TextField(
-            key="CUSTOM_DNS_MANUAL",
-            label="Manual DNS Servers",
-            description="Comma-separated list of DNS server IP addresses (e.g., 8.8.8.8, 1.1.1.1).",
-            placeholder="8.8.8.8, 1.1.1.1",
-            disabled=tor_overrides_network,
-            disabled_reason="DNS is managed by Tor when Tor routing is enabled.",
-            show_when={"field": "CUSTOM_DNS", "value": "manual"},
-        ),
-        CheckboxField(
-            key="USE_DOH",
-            label="Use DNS over HTTPS",
-            description=(
-                "Not applicable when Tor routing is enabled."
-                if tor_overrides_network
-                else "Use encrypted DNS queries for improved reliability and privacy."
-            ),
-            default=True,
-            disabled=tor_overrides_network,
-            disabled_reason="DNS over HTTPS is not used when Tor routing is enabled.",
-            # Hide for manual and system (no DoH endpoint available for custom IPs or system DNS)
-            show_when={"field": "CUSTOM_DNS", "value": ["auto", "google", "cloudflare", "quad9", "opendns"]},
-            # Disable for auto (always uses DoH)
-            disabled_when={
-                "field": "CUSTOM_DNS",
-                "value": "auto",
-                "reason": "Auto mode always uses DNS over HTTPS for reliable provider rotation.",
-            },
-        ),
-        CheckboxField(
-            key="USING_TOR",
-            label="Tor Routing",
-            description=(
-                "All traffic is routed through Tor in this container variant. This cannot be changed."
-                if tor_available
-                else "Tor routing is not available in this container variant."
-            ),
-            default=tor_available,  # Reflects actual state: True if Tor variant, False otherwise
-            disabled=True,  # Always disabled - Tor state is determined by container variant
-            disabled_reason=(
-                "Tor routing is always active in the Tor container variant."
-                if tor_available
-                else "Requires the Tor container variant (calibre-web-automated-book-downloader-tor)."
-            ),
-        ),
-        SelectField(
-            key="PROXY_MODE",
-            label="Proxy Mode",
-            description=(
-                "Not applicable when Tor routing is enabled."
-                if tor_overrides_network
-                else "Choose proxy type. SOCKS5 handles all traffic through a single proxy."
-            ),
-            options=[
-                {"value": "none", "label": "None (Direct Connection)"},
-                {"value": "http", "label": "HTTP/HTTPS Proxy"},
-                {"value": "socks5", "label": "SOCKS5 Proxy"},
-            ],
-            default="none",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-        ),
-        TextField(
-            key="HTTP_PROXY",
-            label="HTTP Proxy",
-            description="HTTP proxy URL (e.g., http://proxy:8080)",
-            placeholder="http://proxy:8080",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-            show_when={"field": "PROXY_MODE", "value": "http"},
-        ),
-        TextField(
-            key="HTTPS_PROXY",
-            label="HTTPS Proxy",
-            description="HTTPS proxy URL (leave empty to use HTTP proxy for HTTPS)",
-            placeholder="http://proxy:8080",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-            show_when={"field": "PROXY_MODE", "value": "http"},
-        ),
-        TextField(
-            key="SOCKS5_PROXY",
-            label="SOCKS5 Proxy",
-            description="SOCKS5 proxy URL. Supports auth: socks5://user:pass@host:port",
-            placeholder="socks5://localhost:1080",
-            disabled=tor_overrides_network,
-            disabled_reason="Proxy settings are not used when Tor routing is enabled.",
-            show_when={"field": "PROXY_MODE", "value": "socks5"},
-        ),
-    ]
-
-
-@register_settings("downloads", "Downloads", icon="folder", order=5)
-def download_settings():
-    """Configure download behavior and file locations."""
-    return [
-        TextField(
-            key="INGEST_DIR",
-            label="Download Directory",
-            description="Directory where downloaded files are saved.",
-            default="/cwa-book-ingest",
-            required=True,
-        ),
-        CheckboxField(
-            key="USE_BOOK_TITLE",
-            label="Use Book Info as Filename",
-            description="Save files using Author, Title and Year instead of ID. May cause issues with special characters.",
-            default=True,
-        ),
-        CheckboxField(
-            key="AUTO_OPEN_DOWNLOADS_SIDEBAR",
-            label="Auto-Open Downloads Sidebar",
-            description="Automatically open the downloads sidebar when a new download is queued.",
-            default=False,
-            env_supported=False,  # UI-only setting
-        ),
-        CheckboxField(
-            key="DOWNLOAD_TO_BROWSER",
-            label="Download to Browser",
-            description="Automatically download completed files to your browser.",
-            default=False,
-            env_supported=False,  # UI-only setting
-        ),
-        NumberField(
-            key="MAX_CONCURRENT_DOWNLOADS",
-            label="Max Concurrent Downloads",
-            description="Maximum number of simultaneous downloads.",
-            default=3,
-            min_value=1,
-            max_value=10,
-            requires_restart=True,
-        ),
-        NumberField(
-            key="STATUS_TIMEOUT",
-            label="Status Timeout (seconds)",
-            description="How long to keep completed/failed downloads in the queue display.",
-            default=3600,
-            min_value=60,
-            max_value=86400,
-        ),
-        CheckboxField(
-            key="USE_CONTENT_TYPE_DIRECTORIES",
-            label="Configure Content-Type Directories",
-            description="Show options to specify custom directories for each content type (fiction, non-fiction, comics, etc.). If a directory is set, that content type will be saved there instead of the default download directory.",
-            default=False,
-            env_supported=False,  # UI-only toggle to show/hide directory fields
-        ),
-        HeadingField(
-            key="content_type_directories_heading",
-            title="Content-Type Directories",
-            description="Specify custom directories for each content type. Leave empty to use the default download directory.",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_BOOK_FICTION",
-            label="Fiction Books",
-            placeholder="/cwa-book-ingest/fiction",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_BOOK_NON_FICTION",
-            label="Non-Fiction Books",
-            placeholder="/cwa-book-ingest/non-fiction",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_BOOK_UNKNOWN",
-            label="Unknown Books",
-            placeholder="/cwa-book-ingest/unknown",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_MAGAZINE",
-            label="Magazines",
-            placeholder="/cwa-book-ingest/magazines",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_COMIC_BOOK",
-            label="Comic Books",
-            placeholder="/cwa-book-ingest/comics",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_AUDIOBOOK",
-            label="Audiobooks",
-            placeholder="/cwa-book-ingest/audiobooks",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_STANDARDS_DOCUMENT",
-            label="Standards Documents",
-            placeholder="/cwa-book-ingest/standards",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_MUSICAL_SCORE",
-            label="Musical Scores",
-            placeholder="/cwa-book-ingest/scores",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-        TextField(
-            key="INGEST_DIR_OTHER",
-            label="Other",
-            placeholder="/cwa-book-ingest/other",
-            show_when={"field": "USE_CONTENT_TYPE_DIRECTORIES", "value": True},
-        ),
-    ]
-
-
-def _get_source_priority_options():
-    """Build source priority options with dynamic disabled states."""
-    from cwa_book_downloader.core.config import config
-
-    has_donator_key = bool(config.get("AA_DONATOR_KEY", ""))
-    use_cf_bypass = config.get("USE_CF_BYPASS", True)
-    using_external_bypasser = config.get("USING_EXTERNAL_BYPASSER", False)
-    has_internal_bypasser = use_cf_bypass and not using_external_bypasser
-
-    return [
-        {
-            "id": "aa-fast",
-            "label": "Anna's Archive (Fast)",
-            "description": "Fast downloads for donators",
-            "isLocked": not has_donator_key,
-            "disabledReason": "Requires AA Donator Key" if not has_donator_key else None,
-        },
-        {
-            "id": "welib",
-            "label": "Welib",
-            "description": "Alternative mirror with good availability",
-            "isLocked": not has_internal_bypasser,
-            "disabledReason": "Requires internal bypasser" if not has_internal_bypasser else None,
-        },
-        {
-            "id": "aa-slow-nowait",
-            "label": "Anna's Archive (Slowest, No Waitlist)",
-            "description": "Partner servers without countdown",
-        },
-        {
-            "id": "aa-slow-wait",
-            "label": "Anna's Archive (Slow, Waitlist)",
-            "description": "Partner servers with countdown timer",
-        },
-        {
-            "id": "libgen",
-            "label": "Libgen",
-            "description": "Library Genesis mirrors",
-        },
-        {
-            "id": "zlib",
-            "label": "Z-Library",
-            "description": "Z-Library mirrors (requires Cloudflare bypass)",
-            "isLocked": not has_internal_bypasser,
-            "disabledReason": "Requires internal bypasser" if not has_internal_bypasser else None,
-        },
-    ]
-
-
-def _get_default_source_priority():
-    """Default source priority order, respecting legacy env vars.
-
-    ALLOW_USE_WELIB (default true) controls whether welib is enabled.
-    PRIORITIZE_WELIB (default false) controls whether welib is moved to position 1.
-    """
-    from cwa_book_downloader.config.env import _LEGACY_PRIORITIZE_WELIB, _LEGACY_ALLOW_USE_WELIB
-
-    welib_entry = {"id": "welib", "enabled": _LEGACY_ALLOW_USE_WELIB}
-
-    priority = [
-        {"id": "aa-fast", "enabled": True},
-        {"id": "aa-slow-nowait", "enabled": True},
-        {"id": "aa-slow-wait", "enabled": True},
-        {"id": "libgen", "enabled": True},
-    ]
-
-    if _LEGACY_PRIORITIZE_WELIB:
-        priority.insert(1, welib_entry)  # After aa-fast
-    else:
-        priority.append(welib_entry)  # Before zlib
-
-    # Z-Library last - it's quite brittle
-    priority.append({"id": "zlib", "enabled": True})
-
-    return priority
-
-
-@register_settings("download_sources", "Download Sources", icon="download", order=21, group="direct_download")
-def download_source_settings():
-    """Settings for download source behavior."""
-    return [
-        HeadingField(
-            key="source_priority_heading",
-            title="Source Priority",
-            description="Configure which download sources to use and in what order.",
-        ),
-        OrderableListField(
-            key="SOURCE_PRIORITY",
-            label="Download Source Order",
-            description="Drag to reorder. Sources are tried from top to bottom until a download succeeds.",
-            options=_get_source_priority_options,
-            default=_get_default_source_priority(),
-        ),
-        NumberField(
-            key="MAX_RETRY",
-            label="Max Retries",
-            description="Maximum retry attempts for failed downloads.",
-            default=10,
-            min_value=1,
-            max_value=50,
-        ),
-        NumberField(
-            key="DEFAULT_SLEEP",
-            label="Retry Delay (seconds)",
-            description="Wait time between download retry attempts.",
-            default=5,
-            min_value=1,
-            max_value=60,
-        ),
-        HeadingField(
-            key="aa_settings_heading",
-            title="Anna's Archive",
-            description="Configure Anna's Archive mirror and donator settings.",
-        ),
-        SelectField(
-            key="AA_BASE_URL",
-            label="Anna's Archive URL",
-            description="Primary Anna's Archive mirror to use. 'auto' selects automatically.",
-            options=[
-                {"value": "auto", "label": "Auto (Recommended)"},
-                {"value": "https://annas-archive.org", "label": "annas-archive.org"},
-                {"value": "https://annas-archive.se", "label": "annas-archive.se"},
-                {"value": "https://annas-archive.li", "label": "annas-archive.li"},
-            ],
-            default="auto",
-        ),
-        TextField(
-            key="AA_ADDITIONAL_URLS",
-            label="Additional AA Mirrors",
-            description="Comma-separated list of additional Anna's Archive mirror URLs.",
-            placeholder="https://example.com,https://another.com",
-        ),
-        PasswordField(
-            key="AA_DONATOR_KEY",
-            label="Anna's Archive Donator Key",
-            description="Optional donator key for faster downloads from Anna's Archive.",
-        ),
-    ]
-
-
-@register_settings("cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download")
-def cloudflare_bypass_settings():
-    """Settings for Cloudflare bypass behavior."""
-    return [
-        CheckboxField(
-            key="USE_CF_BYPASS",
-            label="Enable Cloudflare Bypass",
-            description="Attempt to bypass Cloudflare protection on download sites.",
-            default=True,
-            requires_restart=True,
-        ),
-        CheckboxField(
-            key="BYPASS_WARMUP_ON_CONNECT",
-            label="Warmup on Connect",
-            description="Pre-warm the bypasser when user connects to Web App UI",
-            default=True,
-        ),
-        NumberField(
-            key="BYPASS_RELEASE_INACTIVE_MIN",
-            label="Release Inactive (minutes)",
-            description="Release bypasser resources after this many minutes of inactivity.",
-            default=5,
-            min_value=1,
-            max_value=60,
-        ),
-        CheckboxField(
-            key="USING_EXTERNAL_BYPASSER",
-            label="Use External Bypasser",
-            description="Use FlareSolverr or similar external service instead of built-in bypasser. Caution: May have limitations with custom DNS, Tor and proxies. You may experience slower downloads and and poorer reliability compared to the internal bypasser.",
-            default=False,
-            requires_restart=True,
-        ),
-        TextField(
-            key="EXT_BYPASSER_URL",
-            label="External Bypasser URL",
-            description="URL of the external bypasser service (e.g., FlareSolverr).",
-            default="http://flaresolverr:8191",
-            placeholder="http://flaresolverr:8191",
-            requires_restart=True,
-            show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
-        ),
-        TextField(
-            key="EXT_BYPASSER_PATH",
-            label="External Bypasser Path",
-            description="API path for the external bypasser.",
-            default="/v1",
-            placeholder="/v1",
-            requires_restart=True,
-            show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
-        ),
-        NumberField(
-            key="EXT_BYPASSER_TIMEOUT",
-            label="External Bypasser Timeout (ms)",
-            description="Timeout for external bypasser requests in milliseconds.",
-            default=60000,
-            min_value=10000,
-            max_value=300000,
-            requires_restart=True,
-            show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
-        ),
-    ]
-
-
-@register_settings("advanced", "Advanced", icon="cog", order=15)
-def advanced_settings():
-    """Advanced settings for power users."""
-    return [
-        TextField(
-            key="CUSTOM_SCRIPT",
-            label="Custom Script Path",
-            description="Path to a script to run after each successful download. Must be executable.",
-            placeholder="/path/to/script.sh",
-        ),
-        CheckboxField(
-            key="DEBUG",
-            label="Debug Mode",
-            description="Enable verbose logging to console and file. Not recommended for normal use.",
-            default=False,
-            requires_restart=True,
-        ),
-        NumberField(
-            key="MAIN_LOOP_SLEEP_TIME",
-            label="Queue Check Interval (seconds)",
-            description="How often the download queue is checked for new items.",
-            default=5,
-            min_value=1,
-            max_value=60,
-            requires_restart=True,
-        ),
-        NumberField(
-            key="DOWNLOAD_PROGRESS_UPDATE_INTERVAL",
-            label="Progress Update Interval (seconds)",
-            description="How often download progress is broadcast to the UI.",
-            default=1,
-            min_value=1,
-            max_value=10,
-            requires_restart=True,
-        ),
-        HeadingField(
-            key="covers_cache_heading",
-            title="Cover Image Cache",
-            description="Cache book cover images locally for faster loading. Works for both Direct Download and Universal mode.",
-        ),
-        CheckboxField(
-            key="COVERS_CACHE_ENABLED",
-            label="Enable Cover Cache",
-            description="Cache book covers on the server for faster loading.",
-            default=True,
-        ),
-        NumberField(
-            key="COVERS_CACHE_TTL",
-            label="Cache TTL (days)",
-            description="How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork).",
-            default=0,
-            min_value=0,
-            max_value=365,
-        ),
-        NumberField(
-            key="COVERS_CACHE_MAX_SIZE_MB",
-            label="Max Cache Size (MB)",
-            description="Maximum disk space for cached covers. Oldest images are removed when limit is reached.",
-            default=500,
-            min_value=50,
-            max_value=5000,
-        ),
-        ActionButton(
-            key="clear_covers_cache",
-            label="Clear Cover Cache",
-            description="Delete all cached cover images.",
-            style="danger",
-            callback=_clear_covers_cache,
-        ),
-        HeadingField(
-            key="metadata_cache_heading",
-            title="Metadata Cache",
-            description="Cache book metadata from providers (Hardcover, Open Library) to reduce API calls and speed up repeated searches.",
-        ),
-        CheckboxField(
-            key="METADATA_CACHE_ENABLED",
-            label="Enable Metadata Caching",
-            description="When disabled, all metadata searches hit the provider API directly.",
-            default=True,
-        ),
-        NumberField(
-            key="METADATA_CACHE_SEARCH_TTL",
-            label="Search Results Cache (seconds)",
-            description="How long to cache search results. Default: 300 (5 minutes). Max: 604800 (7 days).",
-            default=300,
-            min_value=60,
-            max_value=604800,
-            show_when={"field": "METADATA_CACHE_ENABLED", "value": True},
-        ),
-        NumberField(
-            key="METADATA_CACHE_BOOK_TTL",
-            label="Book Details Cache (seconds)",
-            description="How long to cache individual book details. Default: 600 (10 minutes). Max: 604800 (7 days).",
-            default=600,
-            min_value=60,
-            max_value=604800,
-            show_when={"field": "METADATA_CACHE_ENABLED", "value": True},
-        ),
-        ActionButton(
-            key="clear_metadata_cache",
-            label="Clear Metadata Cache",
-            description="Clear all cached search results and book details.",
-            style="danger",
-            callback=_clear_metadata_cache,
-        ),
-    ]

cwa_book_downloader/core/__init__.py

@@ -1,5 +0,0 @@
-"""Core module - shared models, queue, and utilities."""
-
-from cwa_book_downloader.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
-from cwa_book_downloader.core.queue import BookQueue, book_queue
-from cwa_book_downloader.core.logger import setup_logger

cwa_book_downloader/download/archive.py

@@ -1,354 +0,0 @@
-"""Archive extraction utilities for downloaded book archives."""
-
-import os
-import shutil
-import zipfile
-from dataclasses import dataclass
-from pathlib import Path
-from typing import List, Optional, Tuple
-
-from cwa_book_downloader.core.logger import setup_logger
-
-logger = setup_logger(__name__)
-
-# Check for rarfile availability at module load
-try:
-    import rarfile
-
-    RAR_AVAILABLE = True
-except ImportError:
-    RAR_AVAILABLE = False
-    logger.warning("rarfile not installed - RAR extraction disabled")
-
-# Book file extensions that should be kept after extraction
-BOOK_EXTENSIONS = frozenset({
-    "epub", "mobi", "azw", "azw3", "pdf", "fb2", "djvu",
-    "cbz", "cbr", "txt", "rtf", "doc", "docx", "lit", "pdb",
-})
-
-
-class ArchiveExtractionError(Exception):
-    """Raised when archive extraction fails."""
-
-    pass
-
-
-class PasswordProtectedError(ArchiveExtractionError):
-    """Raised when archive requires a password."""
-
-    pass
-
-
-class CorruptedArchiveError(ArchiveExtractionError):
-    """Raised when archive is corrupted."""
-
-    pass
-
-
-def is_archive(file_path: Path) -> bool:
-    """Check if file is a supported archive format."""
-    suffix = file_path.suffix.lower().lstrip(".")
-    return suffix in ("zip", "rar")
-
-
-def _is_book_file(file_path: Path) -> bool:
-    """Check if file is a recognized book format."""
-    ext = file_path.suffix.lower().lstrip(".")
-    return ext in BOOK_EXTENSIONS
-
-
-def _filter_book_files(extracted_files: List[Path]) -> Tuple[List[Path], List[Path]]:
-    """
-    Filter extracted files to only book formats.
-
-    Returns:
-        Tuple of (book_files, non_book_files)
-    """
-    book_files = []
-    non_book_files = []
-
-    for file_path in extracted_files:
-        if _is_book_file(file_path):
-            book_files.append(file_path)
-        else:
-            non_book_files.append(file_path)
-
-    return book_files, non_book_files
-
-
-def extract_archive(
-    archive_path: Path,
-    output_dir: Path,
-) -> Tuple[List[Path], List[str]]:
-    """
-    Extract book files from an archive.
-
-    Extracts all files, then filters to only keep recognized book formats.
-    Non-book files (HTML, images, etc.) are deleted.
-
-    Args:
-        archive_path: Path to the archive file
-        output_dir: Directory to extract files to
-
-    Returns:
-        Tuple of (extracted_book_file_paths, warnings)
-
-    Raises:
-        ArchiveExtractionError: If extraction fails
-        PasswordProtectedError: If archive requires password
-        CorruptedArchiveError: If archive is corrupted
-    """
-    suffix = archive_path.suffix.lower().lstrip(".")
-
-    if suffix == "zip":
-        extracted_files, warnings = _extract_zip(archive_path, output_dir)
-    elif suffix == "rar":
-        extracted_files, warnings = _extract_rar(archive_path, output_dir)
-    else:
-        raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
-
-    # Filter to only book files, delete non-book files
-    book_files, non_book_files = _filter_book_files(extracted_files)
-
-    for non_book_file in non_book_files:
-        try:
-            non_book_file.unlink()
-            logger.debug(f"Deleted non-book file: {non_book_file.name}")
-        except OSError as e:
-            logger.warning(f"Failed to delete non-book file {non_book_file}: {e}")
-
-    if non_book_files:
-        warnings.append(f"Skipped {len(non_book_files)} non-book file(s)")
-
-    return book_files, warnings
-
-
-def _extract_zip(
-    archive_path: Path,
-    output_dir: Path,
-) -> Tuple[List[Path], List[str]]:
-    """Extract files from a ZIP archive."""
-    extracted_files = []
-    warnings = []
-
-    try:
-        with zipfile.ZipFile(archive_path, "r") as zf:
-            # Check for password protection
-            for info in zf.infolist():
-                if info.flag_bits & 0x1:  # Encrypted flag
-                    raise PasswordProtectedError("ZIP archive is password protected")
-
-            # Test archive integrity
-            bad_file = zf.testzip()
-            if bad_file:
-                raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
-
-            # Extract all files
-            for info in zf.infolist():
-                if info.is_dir():
-                    continue
-
-                # Use only filename, strip directory path (security: prevent path traversal)
-                filename = Path(info.filename).name
-                if not filename:
-                    continue
-
-                # Extract to output_dir with flat structure
-                target_path = output_dir / filename
-                target_path = _handle_duplicate_filename(target_path)
-
-                with zf.open(info) as src, open(target_path, "wb") as dst:
-                    dst.write(src.read())
-
-                extracted_files.append(target_path)
-                logger.debug(f"Extracted: {filename}")
-
-    except zipfile.BadZipFile as e:
-        raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
-    except PermissionError as e:
-        raise ArchiveExtractionError(f"Permission denied: {e}")
-
-    return extracted_files, warnings
-
-
-def _extract_rar(
-    archive_path: Path,
-    output_dir: Path,
-) -> Tuple[List[Path], List[str]]:
-    """Extract files from a RAR archive."""
-    if not RAR_AVAILABLE:
-        raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
-
-    extracted_files = []
-    warnings = []
-
-    try:
-        with rarfile.RarFile(archive_path, "r") as rf:
-            # Check for password protection
-            if rf.needs_password():
-                raise PasswordProtectedError("RAR archive is password protected")
-
-            # Test archive integrity
-            rf.testrar()
-
-            # Extract all files
-            for info in rf.infolist():
-                if info.is_dir():
-                    continue
-
-                # Use only filename, strip directory path (security: prevent path traversal)
-                filename = Path(info.filename).name
-                if not filename:
-                    continue
-
-                # Extract to output_dir with flat structure
-                target_path = output_dir / filename
-                target_path = _handle_duplicate_filename(target_path)
-
-                with rf.open(info) as src, open(target_path, "wb") as dst:
-                    dst.write(src.read())
-
-                extracted_files.append(target_path)
-                logger.debug(f"Extracted: {filename}")
-
-    except rarfile.BadRarFile as e:
-        raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
-    except rarfile.RarCannotExec:
-        raise ArchiveExtractionError("unrar binary not found - install unrar package")
-    except PermissionError as e:
-        raise ArchiveExtractionError(f"Permission denied: {e}")
-
-    return extracted_files, warnings
-
-
-def _handle_duplicate_filename(target_path: Path) -> Path:
-    """Handle duplicate filenames by appending counter."""
-    if not target_path.exists():
-        return target_path
-
-    base = target_path.stem
-    ext = target_path.suffix
-    parent = target_path.parent
-    counter = 1
-
-    while target_path.exists():
-        target_path = parent / f"{base}_{counter}{ext}"
-        counter += 1
-
-    return target_path
-
-
-@dataclass
-class ArchiveResult:
-    """Result of archive processing."""
-
-    success: bool
-    final_paths: List[Path]
-    message: str
-    error: Optional[str] = None
-
-
-def process_archive(
-    archive_path: Path,
-    temp_dir: Path,
-    ingest_dir: Path,
-    archive_id: str,
-) -> ArchiveResult:
-    """
-    Process an archive file: extract, filter to book files, move to ingest.
-
-    This is the main entry point for archive handling, usable by any download handler.
-
-    Args:
-        archive_path: Path to the downloaded archive file
-        temp_dir: Base temp directory for extraction (e.g., TMP_DIR)
-        ingest_dir: Final destination directory for book files
-        archive_id: Unique identifier for temp directory naming
-
-    Returns:
-        ArchiveResult with success status, final paths, and status message
-    """
-    extract_dir = temp_dir / f"extract_{archive_id}"
-
-    try:
-        # Create temp extraction directory
-        os.makedirs(extract_dir, exist_ok=True)
-        os.makedirs(ingest_dir, exist_ok=True)
-
-        # Extract to temp directory (filters to book files only)
-        extracted_files, warnings = extract_archive(archive_path, extract_dir)
-
-        if not extracted_files:
-            # Clean up and return error
-            shutil.rmtree(extract_dir, ignore_errors=True)
-            archive_path.unlink(missing_ok=True)
-            return ArchiveResult(
-                success=False,
-                final_paths=[],
-                message="",
-                error="No book files found in archive",
-            )
-
-        for warning in warnings:
-            logger.debug(warning)
-
-        logger.info(f"Extracted {len(extracted_files)} book file(s) from archive")
-
-        # Move book files to ingest folder
-        final_paths = []
-        for extracted_file in extracted_files:
-            final_path = ingest_dir / extracted_file.name
-            final_path = _handle_duplicate_filename(final_path)
-            shutil.move(str(extracted_file), str(final_path))
-            final_paths.append(final_path)
-            logger.debug(f"Moved to ingest: {final_path.name}")
-
-        # Clean up temp extraction directory and archive
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-
-        # Build success message with extracted formats
-        formats = [p.suffix.lstrip(".").upper() for p in final_paths]
-        if len(formats) == 1:
-            message = f"Extracted: {formats[0]}"
-        else:
-            message = f"Extracted: {len(formats)} files ({', '.join(formats)})"
-
-        return ArchiveResult(
-            success=True,
-            final_paths=final_paths,
-            message=message,
-        )
-
-    except PasswordProtectedError:
-        logger.error(f"Password-protected archive: {archive_path.name}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-        return ArchiveResult(
-            success=False,
-            final_paths=[],
-            message="",
-            error="Archive is password protected",
-        )
-
-    except CorruptedArchiveError as e:
-        logger.error(f"Corrupted archive: {e}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-        return ArchiveResult(
-            success=False,
-            final_paths=[],
-            message="",
-            error=f"Corrupted archive: {e}",
-        )
-
-    except ArchiveExtractionError as e:
-        logger.error(f"Archive extraction failed: {e}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
-        archive_path.unlink(missing_ok=True)
-        return ArchiveResult(
-            success=False,
-            final_paths=[],
-            message="",
-            error=f"Extraction failed: {e}",
-        )

cwa_book_downloader/download_clients/__init__.py

@@ -1,55 +0,0 @@
-"""External download client integrations (qBittorrent, SABnzbd, etc.)."""
-
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
-from enum import Enum
-
-
-class DownloadStatus(Enum):
-    """Status of a download in an external client."""
-    QUEUED = "queued"
-    DOWNLOADING = "downloading"
-    PAUSED = "paused"
-    COMPLETED = "completed"
-    FAILED = "failed"
-    SEEDING = "seeding"  # Torrents only
-
-
-@dataclass
-class ClientDownloadProgress:
-    """Progress info from external download client."""
-    status: DownloadStatus
-    progress: float                  # 0-100
-    download_speed: Optional[int]    # bytes/sec
-    eta: Optional[int]               # seconds remaining
-    save_path: Optional[str]         # Where the file will be/is
-
-
-class DownloadClient(ABC):
-    """Abstract base class for download clients."""
-
-    @abstractmethod
-    def add_download(self, url: str, title: str) -> str:
-        """Add a download (torrent/magnet or NZB URL). Returns download ID for tracking."""
-        pass
-
-    @abstractmethod
-    def get_download(self, download_id: str) -> Optional[ClientDownloadProgress]:
-        """Get progress of a specific download."""
-        pass
-
-    @abstractmethod
-    def list_downloads(self) -> List[Tuple[str, ClientDownloadProgress]]:
-        """List all downloads with their progress."""
-        pass
-
-    @abstractmethod
-    def get_completed_path(self, download_id: str) -> Optional[str]:
-        """Get the path to completed download."""
-        pass
-
-    @abstractmethod
-    def test_connection(self) -> bool:
-        """Test if the client is reachable and credentials are valid."""
-        pass

cwa_book_downloader/metadata_providers/hardcover.py

@@ -1,752 +0,0 @@
-"""Hardcover.app metadata provider. Requires API key."""
-
-import requests
-from typing import Any, Dict, List, Optional
-
-from cwa_book_downloader.core.cache import cacheable
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.settings_registry import (
-    register_settings,
-    CheckboxField,
-    PasswordField,
-    SelectField,
-    ActionButton,
-    HeadingField,
-)
-from cwa_book_downloader.core.config import config as app_config
-from cwa_book_downloader.metadata_providers import (
-    BookMetadata,
-    DisplayField,
-    MetadataProvider,
-    MetadataSearchOptions,
-    SearchType,
-    SortOrder,
-    register_provider,
-    register_provider_kwargs,
-    TextSearchField,
-)
-
-logger = setup_logger(__name__)
-
-HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql"
-
-
-# Mapping from abstract sort order to Hardcover sort parameter
-# Note: release_year is more consistently populated than release_date_i
-SORT_MAPPING: Dict[SortOrder, str] = {
-    SortOrder.RELEVANCE: "_text_match:desc,users_count:desc",
-    SortOrder.POPULARITY: "users_count:desc",
-    SortOrder.RATING: "rating:desc",
-    SortOrder.NEWEST: "release_year:desc",
-    SortOrder.OLDEST: "release_year:asc",
-}
-
-# Mapping from abstract search type to Hardcover fields parameter
-SEARCH_TYPE_FIELDS: Dict[SearchType, str] = {
-    SearchType.GENERAL: "title,isbns,series_names,author_names,alternative_titles",
-    SearchType.TITLE: "title,alternative_titles",
-    SearchType.AUTHOR: "author_names",
-    # ISBN is handled separately via search_by_isbn()
-}
-
-
-def _combine_headline_description(headline: Optional[str], description: Optional[str]) -> Optional[str]:
-    """Combine headline (tagline) and description into a single description.
-
-    Hardcover stores a short 'headline' (tagline/promotional text) separately
-    from the main description. This combines them for display.
-
-    Args:
-        headline: Short promotional text or tagline.
-        description: Full book synopsis/description.
-
-    Returns:
-        Combined description with headline as the first line, or just one if only one exists.
-    """
-    if headline and description:
-        # Add headline as first paragraph, followed by description
-        return f"{headline}\n\n{description}"
-    elif headline:
-        return headline
-    elif description:
-        return description
-    return None
-
-
-@register_provider_kwargs("hardcover")
-def _hardcover_kwargs() -> Dict[str, Any]:
-    """Provide Hardcover-specific constructor kwargs."""
-    return {"api_key": app_config.get("HARDCOVER_API_KEY", "")}
-
-
-@register_provider("hardcover")
-class HardcoverProvider(MetadataProvider):
-    """Hardcover.app metadata provider using GraphQL API."""
-
-    name = "hardcover"
-    display_name = "Hardcover"
-    requires_auth = True
-    supported_sorts = [
-        SortOrder.RELEVANCE,
-        SortOrder.POPULARITY,
-        SortOrder.RATING,
-        SortOrder.NEWEST,
-        SortOrder.OLDEST,
-    ]
-    search_fields = [
-        TextSearchField(
-            key="author",
-            label="Author",
-            description="Search by author name",
-        ),
-        TextSearchField(
-            key="title",
-            label="Title",
-            description="Search by book title",
-        ),
-    ]
-
-    def __init__(self, api_key: Optional[str] = None):
-        """Initialize provider with API key.
-
-        Args:
-            api_key: Hardcover API key. If not provided, uses config singleton.
-        """
-        self.api_key = api_key or app_config.get("HARDCOVER_API_KEY", "")
-        self.session = requests.Session()
-        if self.api_key:
-            self.session.headers.update({
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json",
-            })
-
-    def is_available(self) -> bool:
-        """Check if provider is configured with an API key."""
-        return bool(self.api_key)
-
-    def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
-        """Search for books using Hardcover's search API.
-
-        Args:
-            options: Search options (query, type, sort, pagination, fields).
-
-        Returns:
-            List of BookMetadata objects.
-        """
-        if not self.api_key:
-            logger.warning("Hardcover API key not configured")
-            return []
-
-        # Handle ISBN search separately
-        if options.search_type == SearchType.ISBN:
-            result = self.search_by_isbn(options.query)
-            return [result] if result else []
-
-        # Build cache key from options (include fields for cache differentiation)
-        fields_key = ":".join(f"{k}={v}" for k, v in sorted(options.fields.items()))
-        cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.limit}:{options.page}:{fields_key}"
-        return self._search_cached(cache_key, options)
-
-    @cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="hardcover:search")
-    def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]:
-        """Cached search implementation.
-
-        Args:
-            cache_key: Cache key (used by decorator).
-            options: Search options.
-
-        Returns:
-            List of BookMetadata objects.
-        """
-        # Determine query and fields based on custom search fields
-        # Field-first search: when a specific field has a value, search that field
-        author_value = options.fields.get("author", "").strip()
-        title_value = options.fields.get("title", "").strip()
-
-        logger.debug(f"Field-first search check: author_value='{author_value}', title_value='{title_value}'")
-
-        # Determine what to search and which fields to target
-        # Note: Hardcover API requires 'weights' when using 'fields' parameter
-        if author_value and not title_value:
-            # Author-only search: search author_names field with author query
-            query = author_value
-            search_fields = "author_names"
-            search_weights = "1"
-            logger.debug(f"Author-only search: query='{query}', fields='{search_fields}'")
-        elif title_value and not author_value:
-            # Title-only search: search title fields with title query
-            query = title_value
-            search_fields = "title,alternative_titles"
-            search_weights = "5,1"
-            logger.debug(f"Title-only search: query='{query}', fields='{search_fields}'")
-        elif author_value and title_value:
-            # Both provided: combine into query, search both fields
-            query = f"{title_value} {author_value}"
-            search_fields = "title,alternative_titles,author_names"
-            search_weights = "5,1,3"
-            logger.debug(f"Combined search: query='{query}', fields='{search_fields}'")
-        else:
-            # No custom fields: use general query with all default fields
-            query = options.query
-            search_fields = None
-            search_weights = None
-            logger.debug(f"General search: query='{query}', no field restriction")
-
-        # Build GraphQL query with optional fields/weights parameters
-        if search_fields:
-            graphql_query = """
-            query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) {
-                search(
-                    query: $query,
-                    query_type: "Book",
-                    per_page: $limit,
-                    page: $page,
-                    sort: $sort,
-                    fields: $fields,
-                    weights: $weights
-                ) {
-                    results
-                }
-            }
-            """
-        else:
-            graphql_query = """
-            query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) {
-                search(
-                    query: $query,
-                    query_type: "Book",
-                    per_page: $limit,
-                    page: $page,
-                    sort: $sort
-                ) {
-                    results
-                }
-            }
-            """
-
-        # Map abstract sort order to Hardcover's sort parameter
-        sort_param = SORT_MAPPING.get(options.sort, SORT_MAPPING[SortOrder.RELEVANCE])
-
-        variables = {
-            "query": query,
-            "limit": options.limit,
-            "page": options.page,
-            "sort": sort_param,
-        }
-
-        if search_fields:
-            variables["fields"] = search_fields
-            variables["weights"] = search_weights
-
-        logger.debug(f"GraphQL variables: {variables}")
-
-        try:
-            result = self._execute_query(graphql_query, variables)
-            if not result:
-                logger.debug("Hardcover search: No result from API")
-                return []
-
-            search_data = result.get("search", {})
-
-            # Results is a Typesense response object with hits array
-            results_obj = search_data.get("results", {})
-            if isinstance(results_obj, dict):
-                hits = results_obj.get("hits", [])
-            else:
-                hits = results_obj if isinstance(results_obj, list) else []
-
-            # Parse the search results - each hit has a 'document' field
-            books = []
-            for hit in hits:
-                # Get the document from the hit
-                item = hit.get("document", hit) if isinstance(hit, dict) else hit
-                if isinstance(item, dict):
-                    book = self._parse_search_result(item)
-                    if book:
-                        books.append(book)
-
-            logger.info(f"Hardcover search '{query}' (fields={search_fields}) returned {len(books)} results")
-            return books
-
-        except Exception as e:
-            logger.error(f"Hardcover search error: {e}")
-            return []
-
-    @cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="hardcover:book")
-    def get_book(self, book_id: str) -> Optional[BookMetadata]:
-        """Get book details by Hardcover ID.
-
-        Args:
-            book_id: Hardcover book ID.
-
-        Returns:
-            BookMetadata or None if not found.
-        """
-        if not self.api_key:
-            logger.warning("Hardcover API key not configured")
-            return None
-
-        # Query for specific book by ID
-        # Note: API has max depth of 3, so use cached_* fields instead of nested relationships
-        graphql_query = """
-        query GetBook($id: Int!) {
-            books(where: {id: {_eq: $id}}, limit: 1) {
-                id
-                title
-                slug
-                release_date
-                headline
-                description
-                pages
-                cached_image
-                cached_contributors
-                cached_tags
-                default_physical_edition {
-                    isbn_10
-                    isbn_13
-                }
-            }
-        }
-        """
-
-        try:
-            book_id_int = int(book_id)
-            result = self._execute_query(graphql_query, {"id": book_id_int})
-            if not result:
-                return None
-
-            books = result.get("books", [])
-            if not books:
-                return None
-
-            return self._parse_book(books[0])
-
-        except ValueError:
-            logger.error(f"Invalid book ID: {book_id}")
-            return None
-        except Exception as e:
-            logger.error(f"Hardcover get_book error: {e}")
-            return None
-
-    @cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="hardcover:isbn")
-    def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
-        """Search for a book by ISBN.
-
-        Args:
-            isbn: ISBN-10 or ISBN-13.
-
-        Returns:
-            BookMetadata or None if not found.
-        """
-        if not self.api_key:
-            logger.warning("Hardcover API key not configured")
-            return None
-
-        # Clean ISBN (remove hyphens)
-        clean_isbn = isbn.replace("-", "").strip()
-
-        # Search for editions with matching ISBN
-        # Note: API has max depth of 3, so use cached_* fields instead of nested relationships
-        graphql_query = """
-        query SearchByISBN($isbn: String!) {
-            editions(
-                where: {
-                    _or: [
-                        {isbn_10: {_eq: $isbn}},
-                        {isbn_13: {_eq: $isbn}}
-                    ]
-                },
-                limit: 1
-            ) {
-                isbn_10
-                isbn_13
-                book {
-                    id
-                    title
-                    slug
-                    release_date
-                    headline
-                    description
-                    pages
-                    cached_image
-                    cached_contributors
-                    cached_tags
-                }
-            }
-        }
-        """
-
-        try:
-            result = self._execute_query(graphql_query, {"isbn": clean_isbn})
-            if not result:
-                return None
-
-            editions = result.get("editions", [])
-            if not editions:
-                logger.debug(f"No Hardcover book found for ISBN: {isbn}")
-                return None
-
-            edition = editions[0]
-            book_data = edition.get("book", {})
-            if not book_data:
-                return None
-
-            # Add ISBN data from edition to book data
-            book_data["isbn_10"] = edition.get("isbn_10")
-            book_data["isbn_13"] = edition.get("isbn_13")
-
-            return self._parse_book(book_data)
-
-        except Exception as e:
-            logger.error(f"Hardcover ISBN search error: {e}")
-            return None
-
-    def _execute_query(self, query: str, variables: Dict[str, Any]) -> Optional[Dict]:
-        """Execute a GraphQL query.
-
-        Args:
-            query: GraphQL query string.
-            variables: Query variables.
-
-        Returns:
-            Response data dict or None on error.
-        """
-        try:
-            response = self.session.post(
-                HARDCOVER_API_URL,
-                json={"query": query, "variables": variables},
-                timeout=15
-            )
-            response.raise_for_status()
-
-            data = response.json()
-
-            if "errors" in data:
-                logger.error(f"GraphQL errors: {data['errors']}")
-                return None
-
-            return data.get("data")
-
-        except requests.Timeout:
-            logger.warning("Hardcover API request timed out")
-            return None
-        except requests.HTTPError as e:
-            if e.response.status_code == 401:
-                logger.error("Hardcover API key is invalid")
-            else:
-                logger.error(f"Hardcover API HTTP error: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"Hardcover API request failed: {e}")
-            return None
-
-    def _parse_search_result(self, item: Dict) -> Optional[BookMetadata]:
-        """Parse a search result item into BookMetadata.
-
-        Args:
-            item: Search result item dict.
-
-        Returns:
-            BookMetadata or None if parsing fails.
-        """
-        try:
-            book_id = item.get("id") or item.get("document", {}).get("id")
-            title = item.get("title") or item.get("document", {}).get("title")
-
-            if not book_id or not title:
-                return None
-
-            # Extract authors from various possible fields
-            authors = []
-            if "author_names" in item:
-                authors = item["author_names"] if isinstance(item["author_names"], list) else [item["author_names"]]
-            elif "cached_contributors" in item:
-                for contrib in item.get("cached_contributors", []):
-                    if isinstance(contrib, dict) and contrib.get("name"):
-                        authors.append(contrib["name"])
-                    elif isinstance(contrib, str):
-                        authors.append(contrib)
-
-            # Get cover URL
-            cover_url = None
-            if "image" in item and item["image"]:
-                cover_url = item["image"] if isinstance(item["image"], str) else item["image"].get("url")
-
-            # Extract year - prefer release_year if available, fall back to release_date
-            publish_year = None
-            if "release_year" in item and item["release_year"]:
-                try:
-                    publish_year = int(item["release_year"])
-                except (ValueError, TypeError):
-                    pass
-            elif "release_date" in item and item["release_date"]:
-                try:
-                    publish_year = int(str(item["release_date"])[:4])
-                except (ValueError, TypeError):
-                    pass
-
-            slug = item.get("slug", "")
-            source_url = f"https://hardcover.app/books/{slug}" if slug else None
-
-            # Build display fields from Hardcover-specific data
-            display_fields = []
-
-            # Rating (e.g., "4.5 (3,764)")
-            rating = item.get("rating")
-            ratings_count = item.get("ratings_count")
-            if rating is not None:
-                rating_str = f"{rating:.1f}"
-                if ratings_count:
-                    rating_str += f" ({ratings_count:,})"
-                display_fields.append(DisplayField(label="Rating", value=rating_str, icon="star"))
-
-            # Readers (users who have this book)
-            users_count = item.get("users_count")
-            if users_count:
-                display_fields.append(DisplayField(label="Readers", value=f"{users_count:,}", icon="users"))
-
-            # Combine headline and description if both present
-            headline = item.get("headline")
-            description = item.get("description")
-            full_description = _combine_headline_description(headline, description)
-
-            return BookMetadata(
-                provider="hardcover",
-                provider_id=str(book_id),
-                title=title,
-                provider_display_name="Hardcover",
-                authors=authors,
-                cover_url=cover_url,
-                description=full_description,
-                publish_year=publish_year,
-                source_url=source_url,
-                display_fields=display_fields,
-            )
-
-        except Exception as e:
-            logger.debug(f"Failed to parse Hardcover search result: {e}")
-            return None
-
-    def _parse_book(self, book: Dict) -> BookMetadata:
-        """Parse a book object into BookMetadata.
-
-        Args:
-            book: Book data dict from GraphQL response.
-
-        Returns:
-            BookMetadata object.
-        """
-        # Extract authors from cached_contributors (json array) or contributions relationship
-        authors = []
-        if book.get("cached_contributors"):
-            for contrib in book["cached_contributors"]:
-                if isinstance(contrib, dict) and contrib.get("name"):
-                    authors.append(contrib["name"])
-                elif isinstance(contrib, str):
-                    authors.append(contrib)
-        elif book.get("contributions"):
-            # Fallback for contributions relationship (if used)
-            for contrib in book["contributions"]:
-                author = contrib.get("author", {})
-                if author and author.get("name"):
-                    authors.append(author["name"])
-
-        # Get cover URL from cached_image (jsonb) or image relationship
-        cover_url = None
-        if book.get("cached_image"):
-            cached = book["cached_image"]
-            if isinstance(cached, dict):
-                cover_url = cached.get("url")
-            elif isinstance(cached, str):
-                cover_url = cached
-        elif book.get("image"):
-            img = book["image"]
-            cover_url = img if isinstance(img, str) else img.get("url")
-
-        # Extract year from release_date
-        publish_year = None
-        if book.get("release_date"):
-            try:
-                publish_year = int(str(book["release_date"])[:4])
-            except (ValueError, TypeError):
-                pass
-
-        # Extract genres from cached_tags
-        genres = []
-        for tag in book.get("cached_tags", []):
-            if isinstance(tag, dict) and tag.get("tag"):
-                genres.append(tag["tag"])
-            elif isinstance(tag, str):
-                genres.append(tag)
-
-        # Get ISBN from direct fields, default_physical_edition, or editions
-        isbn_10 = book.get("isbn_10")
-        isbn_13 = book.get("isbn_13")
-
-        if not isbn_10 and not isbn_13:
-            # Try default_physical_edition first
-            edition = book.get("default_physical_edition")
-            if edition:
-                isbn_10 = edition.get("isbn_10")
-                isbn_13 = edition.get("isbn_13")
-
-            # Fallback to editions array
-            if not isbn_10 and not isbn_13 and book.get("editions"):
-                for ed in book["editions"]:
-                    if not isbn_10 and ed.get("isbn_10"):
-                        isbn_10 = ed["isbn_10"]
-                    if not isbn_13 and ed.get("isbn_13"):
-                        isbn_13 = ed["isbn_13"]
-                    if isbn_10 and isbn_13:
-                        break
-
-        slug = book.get("slug", "")
-        source_url = f"https://hardcover.app/books/{slug}" if slug else None
-
-        # Combine headline and description if both present
-        headline = book.get("headline")
-        description = book.get("description")
-        full_description = _combine_headline_description(headline, description)
-
-        return BookMetadata(
-            provider="hardcover",
-            provider_id=str(book["id"]),
-            title=book["title"],
-            provider_display_name="Hardcover",
-            authors=authors,
-            isbn_10=isbn_10,
-            isbn_13=isbn_13,
-            cover_url=cover_url,
-            description=full_description,
-            publish_year=publish_year,
-            genres=genres,
-            source_url=source_url,
-        )
-
-
-def _test_hardcover_connection() -> Dict[str, Any]:
-    """Test the Hardcover API connection."""
-    from cwa_book_downloader.core.config import config as app_config
-    from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file
-    from cwa_book_downloader.metadata_providers import get_provider_kwargs
-
-    # Refresh config to pick up any recently saved settings
-    app_config.refresh()
-
-    kwargs = get_provider_kwargs("hardcover")
-    api_key = kwargs.get("api_key")
-
-    # Debug: log key info
-    key_len = len(api_key) if api_key else 0
-    key_preview = f"{api_key[:10]}...{api_key[-10:]}" if key_len > 20 else "(too short)"
-    logger.info(f"Hardcover test: key length={key_len}, preview={key_preview}")
-
-    if not api_key:
-        # Clear any stored username since there's no key
-        _save_connected_username(None)
-        return {"success": False, "message": "No API key configured. Save your key and try again."}
-
-    if key_len < 100:
-        return {"success": False, "message": f"API key seems too short ({key_len} chars). Expected 500+ chars."}
-
-    try:
-        provider = HardcoverProvider(api_key=api_key)
-        # Use the 'me' query to test connection (recommended by API docs)
-        result = provider._execute_query(
-            "query { me { id, username } }",
-            {}
-        )
-        if result is not None:
-            # Handle both single object and array response formats
-            me_data = result.get("me", {})
-            if isinstance(me_data, list) and me_data:
-                me_data = me_data[0]
-            username = me_data.get("username", "Unknown") if isinstance(me_data, dict) else "Unknown"
-
-            # Save the username for persistent display
-            _save_connected_username(username)
-
-            return {"success": True, "message": f"Connected as: {username}"}
-        else:
-            _save_connected_username(None)
-            return {"success": False, "message": "API request failed - check your API key"}
-    except Exception as e:
-        logger.exception("Hardcover connection test failed")
-        _save_connected_username(None)
-        return {"success": False, "message": f"Connection failed: {str(e)}"}
-
-
-def _save_connected_username(username: Optional[str]) -> None:
-    """Save or clear the connected username in config."""
-    from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file
-
-    config = load_config_file("hardcover")
-    if username:
-        config["_connected_username"] = username
-    else:
-        config.pop("_connected_username", None)
-    save_config_file("hardcover", config)
-
-
-def _get_connected_username() -> Optional[str]:
-    """Get the stored connected username."""
-    from cwa_book_downloader.core.settings_registry import load_config_file
-
-    config = load_config_file("hardcover")
-    return config.get("_connected_username")
-
-
-# Hardcover sort options for settings UI
-_HARDCOVER_SORT_OPTIONS = [
-    {"value": "relevance", "label": "Most relevant"},
-    {"value": "popularity", "label": "Most popular"},
-    {"value": "rating", "label": "Highest rated"},
-    {"value": "newest", "label": "Newest"},
-    {"value": "oldest", "label": "Oldest"},
-]
-
-
-@register_settings("hardcover", "Hardcover", icon="book", order=51, group="metadata_providers")
-def hardcover_settings():
-    """Hardcover metadata provider settings."""
-    # Check for connected username to show status
-    connected_user = _get_connected_username()
-    test_button_description = f"Connected as: {connected_user}" if connected_user else "Verify your API key works"
-
-    return [
-        HeadingField(
-            key="hardcover_heading",
-            title="Hardcover",
-            description="A modern book tracking and discovery platform with a comprehensive API.",
-            link_url="https://hardcover.app",
-            link_text="hardcover.app",
-        ),
-        CheckboxField(
-            key="HARDCOVER_ENABLED",
-            label="Enable Hardcover",
-            description="Enable Hardcover as a metadata provider for book searches",
-            default=False,
-        ),
-        PasswordField(
-            key="HARDCOVER_API_KEY",
-            label="API Key",
-            description="Get your API key from hardcover.app/account/api",
-            required=True,
-            env_supported=False,  # UI-only setting, no ENV var support
-        ),
-        ActionButton(
-            key="test_connection",
-            label="Test Connection",
-            description=test_button_description,
-            style="primary",
-            callback=_test_hardcover_connection,
-        ),
-        SelectField(
-            key="HARDCOVER_DEFAULT_SORT",
-            label="Default Sort Order",
-            description="Default sort order for Hardcover search results.",
-            options=_HARDCOVER_SORT_OPTIONS,
-            default="relevance",
-            env_supported=False,  # UI-only setting
-        ),
-    ]

docker-compose.bypass-test.yml

@@ -0,0 +1,39 @@
+# Bypass testing - switch between dev build and v1.0.1
+# Usage:
+#   Test dev build:  docker compose -f docker-compose.bypass-test.yml up shelfmark-dev
+#   Test v1.0.1:     docker compose -f docker-compose.bypass-test.yml up shelfmark-stable
+#   Pull latest dev: docker compose -f docker-compose.bypass-test.yml build shelfmark-dev
+#   Pull v1.0.1:     docker compose -f docker-compose.bypass-test.yml pull shelfmark-stable
+
+services:
+  # Dev image from registry
+  shelfmark-dev:
+    image: ghcr.io/calibrain/shelfmark:dev
+    container_name: shelfmark-bypass-dev
+    environment:
+      PUID: 1000
+      PGID: 1000
+      DEBUG: true
+    ports:
+      - 8084:8084
+    volumes:
+      - ./.local/bypass-test/config-dev:/config
+      - ./.local/bypass-test/books:/books
+      - ./.local/bypass-test/log-dev:/var/log/shelfmark
+      - ./.local/bypass-test/tmp:/tmp/shelfmark
+
+  # Stable v1.0.1 for comparison
+  shelfmark-stable:
+    image: ghcr.io/calibrain/shelfmark:1.0.1
+    container_name: shelfmark-bypass-stable
+    environment:
+      PUID: 1000
+      PGID: 1000
+      DEBUG: true
+    ports:
+      - 8085:8084
+    volumes:
+      - ./.local/bypass-test/config-stable:/config
+      - ./.local/bypass-test/books:/books
+      - ./.local/bypass-test/log-stable:/var/log/shelfmark
+      - ./.local/bypass-test/tmp:/tmp/shelfmark

docker-compose.dev.extbp.yml

@@ -0,0 +1,25 @@
+# Local development - External bypasser variant (lite)
+services:
+  shelfmark-lite-dev:
+    extends:
+      file: ./compose/docker-compose.lite.yml
+      service: shelfmark-lite
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark-lite
+    environment:
+      DEBUG: true
+      EXT_BYPASSER_URL: http://flaresolverr:8191
+      EXT_BYPASSER_PATH: /v1
+      EXT_BYPASSER_TIMEOUT: 60000
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads
+
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.dev.tor.yml

@@ -0,0 +1,20 @@
+# Local development - Tor variant
+services:
+  shelfmark-tor-dev:
+    extends:
+      file: ./compose/docker-compose.tor.yml
+      service: shelfmark-tor
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    environment:
+      DEBUG: true
+      USING_TOR: true
+    volumes:
+      - ./.local/config:/config
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.dev.yml

@@ -1,17 +1,22 @@
 # Local development - builds from source with debug enabled
 services:
-  calibre-web-automated-book-downloader-dev:
+  shelfmark-dev:
     extends:
-      file: ./docker-compose.yml
-      service: calibre-web-automated-book-downloader
+      file: ./compose/docker-compose.yml
+      service: shelfmark
     build:
       context: .
       dockerfile: Dockerfile
-      target: cwa-bd
+      target: shelfmark
+    cap_add:
+      - SYS_PTRACE
     environment:
       DEBUG: true
     volumes:
       - ./.local/config:/config
-      - ./.local/ingest:/cwa-book-ingest
-      - ./.local/log:/var/log/cwa-book-downloader
-      - ./.local/tmp:/tmp/cwa-book-downloader
+      - ./.local/books:/books
+      - ./.local/log:/var/log/shelfmark
+      - ./.local/tmp:/tmp/shelfmark
+      - ./shelfmark:/app/shelfmark:ro
+      # Required for torrent / usenet - path must match your download client's volume exactly
+      # - /path/to/downloads:/path/to/downloads

docker-compose.extbp.dev.yml

@@ -1,23 +0,0 @@
-# Local development - External bypasser variant
-services:
-  calibre-web-automated-book-downloader-extbp-dev:
-    extends:
-      file: ./docker-compose.extbp.yml
-      service: calibre-web-automated-book-downloader-extbp
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-extbp
-    environment:
-      DEBUG: true
-      EXT_BYPASSER_URL: http://flaresolverr:8191
-      EXT_BYPASSER_PATH: /v1
-      EXT_BYPASSER_TIMEOUT: 60000
-    volumes:
-      - ./.local/config:/config
-      - ./.local/ingest:/cwa-book-ingest
-      - ./.local/log:/var/log/cwa-book-downloader
-      - ./.local/tmp:/tmp/cwa-book-downloader
-
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.extbp.yml

@@ -1,20 +0,0 @@
-# Uses external Cloudflare bypasser (FlareSolverr/ByParr) instead of built-in Selenium
-services:
-  calibre-web-automated-book-downloader-extbp:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-extbp:latest
-    environment:
-      TZ: America/New_York
-      EXT_BYPASSER_URL: http://flaresolverr:8191
-      # UID: 1000
-      # GID: 100
-      # CWA_DB_PATH: /auth/app.db
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      - /path/to/config:/config
-      # - /cwa/config/path/app.db:/auth/app.db:ro
-
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest

docker-compose.test-clients.yml

@@ -0,0 +1,181 @@
+# Test stack for download client development
+# Includes shelfmark + all download clients on same network with shared volumes
+#
+# Usage:
+#   docker compose -f docker-compose.test-clients.yml up -d
+#   # Access shelfmark at http://localhost:8084
+#   # Configure clients in Settings > Prowlarr > Download Clients
+#
+# Web UIs:
+#   - shelfmark:    http://localhost:8084
+#   - Prowlarr:     http://localhost:9696 (no auth by default)
+#   - qBittorrent:  http://localhost:8080 (check container logs for temp password)
+#   - Transmission: http://localhost:9091 (admin / admin)
+#   - Deluge:       http://localhost:8112 (password: deluge)
+#   - NZBGet:       http://localhost:6789 (nzbget / tegbzn6789)
+#   - SABnzbd:      http://localhost:8085 (complete setup wizard for API key)
+#   - rTorrent:     http://localhost:8000 (admin / admin - if auth enabled)
+#
+
+services:
+  shelfmark:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: shelfmark
+    container_name: test-shelfmark
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      # All client configuration is done via Settings UI
+      # Use Docker service names for URLs:
+      #   - qBittorrent: http://qbittorrent:8080
+      #   - Transmission: http://transmission:9091
+      #   - Deluge Web UI: http://deluge:8112
+      #   - NZBGet: http://nzbget:6789
+      #   - SABnzbd: http://sabnzbd:8080
+      #   - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
+    ports:
+      - "8084:8084"
+    volumes:
+      # Config and state
+      - ./.local/test-clients/shelfmark/config:/config
+      - ./.local/test-clients/shelfmark/log:/var/log/shelfmark
+      # Book destination directory (where completed books go)
+      - ./.local/test-clients/books:/books
+      # Staging directory
+      - ./.local/test-clients/tmp:/tmp/shelfmark
+      # CRITICAL: Mount client download directories so shelfmark can access completed files
+      - ./.local/test-clients/downloads:/downloads
+      # Mount source code for hot-reload (no rebuild needed for Python changes)
+      - ./shelfmark:/app/shelfmark:ro
+      # Mount tests for running pytest in container
+      - ./tests:/app/tests:ro
+      - ./pyproject.toml:/app/pyproject.toml:ro
+      # Mount client configs for integration tests to read credentials
+      - ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
+      - ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
+    depends_on:
+      - nzbget
+      - sabnzbd
+      - qbittorrent
+      - transmission
+      - deluge
+      - rtorrent
+    restart: unless-stopped
+
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:latest
+    container_name: test-prowlarr
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/prowlarr/config:/config
+    ports:
+      - "9696:9696"
+    restart: unless-stopped
+
+  nzbget:
+    image: lscr.io/linuxserver/nzbget:latest
+    container_name: test-nzbget
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/nzbget/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/nzbget/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "6789:6789"  # Web UI / JSON-RPC
+    restart: unless-stopped
+
+  sabnzbd:
+    image: lscr.io/linuxserver/sabnzbd:latest
+    container_name: test-sabnzbd
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/sabnzbd/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8085:8080"  # Web UI (external:internal)
+    restart: unless-stopped
+
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: test-qbittorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - WEBUI_PORT=8080
+    volumes:
+      - ./.local/test-clients/qbittorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+      - ./.local/test-clients/qbittorrent/custom-cont-init.d:/custom-cont-init.d:ro
+    ports:
+      - "8080:8080"  # Web UI / API
+      - "6882:6881"
+      - "6882:6881/udp"
+    restart: unless-stopped
+
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: test-transmission
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - USER=admin
+      - PASS=admin
+    volumes:
+      - ./.local/test-clients/transmission/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "9091:9091"  # Web UI / RPC
+      - "51413:51413"
+      - "51413:51413/udp"
+    restart: unless-stopped
+
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: test-deluge
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+      - DELUGE_LOGLEVEL=error
+    volumes:
+      - ./.local/test-clients/deluge/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8112:8112"  # Web UI
+      - "58846:58846"  # Daemon RPC
+      - "6881:6881"
+      - "6881:6881/udp"
+    restart: unless-stopped
+
+  rtorrent:
+    image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
+    container_name: test-rtorrent
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=UTC
+    volumes:
+      - ./.local/test-clients/rtorrent/config:/config
+      - ./.local/test-clients/downloads:/downloads
+    ports:
+      - "8000:8000"  # XMLRPC
+      - "8089:8080"  # ruTorrent Web UI
+      - "9000:9000"  # SCGI port
+      - "50000:50000"  # Incoming connections
+      - "6881:6881/udp"
+    restart: unless-stopped

docker-compose.tor.dev.yml

@@ -1,17 +0,0 @@
-# Local development - Tor variant
-services:
-  calibre-web-automated-book-downloader-tor-dev:
-    extends:
-      file: ./docker-compose.tor.yml
-      service: calibre-web-automated-book-downloader-tor
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: cwa-bd-tor
-    environment:
-      DEBUG: true
-    volumes:
-      - ./.local/config:/config
-      - ./.local/ingest:/cwa-book-ingest
-      - ./.local/log:/var/log/cwa-book-downloader
-      - ./.local/tmp:/tmp/cwa-book-downloader

docker-compose.tor.yml

@@ -1,19 +0,0 @@
-# Routes all traffic through Tor - requires NET_ADMIN capability
-services:
-  calibre-web-automated-book-downloader-tor:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest
-    environment:
-      FLASK_PORT: 8084
-      TZ: America/New_York
-      USING_TOR: true
-      # CWA_DB_PATH: /auth/app.db
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest
-      - /path/to/config:/config
-      # - /cwa/config/path/app.db:/auth/app.db:ro

docker-compose.yml

@@ -1,15 +0,0 @@
-services:
-  calibre-web-automated-book-downloader:
-    image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
-    container_name: calibre-web-automated-book-downloader
-    environment:
-      TZ: America/New_York
-      # UID: 1000
-      # GID: 100
-      # CWA_DB_PATH: /auth/app.db
-    ports:
-      - 8084:8084
-    restart: unless-stopped
-    volumes:
-      - /tmp/data/calibre-web/ingest:/cwa-book-ingest # This is where the books will be downloaded and ingested by your book management application
-      - /path/to/config:/config # Configuration files and database

docs/configuration.md

@@ -0,0 +1,125 @@
+# Directory and Volume Setup
+
+This guide explains how to configure directories and Docker volumes for Shelfmark. It focuses on the difference between the destination folder and your download client paths, and how to make those paths line up inside containers.
+
+## Conceptual Overview
+
+```
+DIRECT DOWNLOADS
+
+Shelfmark downloads directly -> destination
+
+TORRENT / USENET
+
+Prowlarr -> Download client saves to <client path>
+         -> Shelfmark reads from <client path>
+         -> Shelfmark processes to destination
+```
+
+Key point: For torrent and usenet downloads, Shelfmark must see the same file path that your download client reports. The container path must match in both containers.
+
+## Direct Download Setup
+
+Direct downloads do not use an external download client. A simple two-folder setup is enough.
+
+Required volumes:
+
+| Container path | Purpose | Notes |
+| --- | --- | --- |
+| `/config` | Settings, database, cover cache | Configurable via `CONFIG_DIR` |
+| `/books` | Destination folder for completed files | Configurable via `INGEST_DIR` and Settings -> Downloads -> Destination |
+
+Example `docker-compose`:
+
+```yaml
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    volumes:
+      - /path/to/config:/config
+      - /path/to/books:/books
+```
+
+Notes:
+- Point `/books` to your library ingest folder (Calibre-Web, Booklore, Audiobookshelf, etc) for automatic import.
+- If you set Books Output Mode to Booklore (API), books are uploaded via API instead of written to `/books`. Audiobooks still use a destination folder.
+- Ensure `PUID`/`PGID` (or legacy `UID`/`GID`) match the owner of the host directories to avoid permission errors.
+
+## Torrent / Usenet Setup
+
+For torrents and usenet, your download client reports a path (for example `/data/torrents/books/MyBook.epub`). Shelfmark must be able to read that exact path inside its own container.
+
+Required volumes:
+
+| Container path | Purpose | Notes |
+| --- | --- | --- |
+| `/config` | Settings, database, cover cache | Configurable via `CONFIG_DIR` |
+| `/books` | Destination folder for processed files | Configurable via `INGEST_DIR` |
+| `<client path>` | Download client path | Must match the download client container path exactly |
+
+Side-by-side example with qBittorrent:
+
+```yaml
+services:
+  shelfmark:
+    volumes:
+      - /path/to/config:/config
+      - /path/to/books:/books
+      - /path/to/downloads:/data/torrents  # Must match client
+
+  qbittorrent:
+    volumes:
+      - /path/to/downloads:/data/torrents  # Same container path
+```
+
+Host paths can be anything. The container path (for example `/data/torrents`) must be identical in both containers.
+
+### Remote Path Mappings
+
+If paths cannot match (different machines or a fixed setup), use Remote Path Mappings.
+
+Where to configure:
+- Settings -> Advanced -> Remote Path Mappings
+
+Example:
+- Client reports `/data/torrents/books/...`
+- Shelfmark can see the same files at `/downloads/books/...`
+- Add a mapping from Remote Path `/data/torrents` to Local Path `/downloads`
+
+## File Processing Options
+
+### Transfer Method (Torrent / Usenet Only)
+
+Available methods:
+- Copy (default). Works everywhere.
+- Hardlink. Preserves seeding without duplicating files.
+
+Hardlink requirements and behavior:
+- Source and destination must be on the same filesystem.
+- If hardlinking is enabled but not possible, Shelfmark falls back to copying.
+- Archive extraction is disabled while hardlinking is enabled.
+- Do not use hardlinking if your destination is a library ingest folder.
+
+### File Organization
+
+Shelfmark supports three organization modes for the destination:
+- None. Keep original filenames from the source.
+- Rename Only. Rename files using a template.
+- Rename and Organize. Create folders and rename using templates. Do not use with ingest folders.
+
+Configure templates in Settings -> Downloads. Template syntax details are documented separately.
+
+## Common Mistakes
+
+- "Download failed - file not found": Path mismatch between Shelfmark and the download client. Ensure container paths match or use Remote Path Mappings.
+- "Permission denied": `PUID`/`PGID` do not match the host directories. Ensure Shelfmark can read the client path and write to the destination.
+- "Hardlinks not working" or "Files being copied instead": Source and destination are on different filesystems. Move the destination or accept copy fallback.
+- "Downloads work but library does not see them": Destination does not point to the library ingest folder. Check Settings -> Downloads -> Destination.
+- CIFS/SMB shares: Use the `nobrl` mount option to avoid database lock errors. Example: `//server/share /mnt/share cifs nobrl,... 0 0`
+
+## Related Documentation
+
+- Environment Variables Reference: `docs/environment-variables.md`
+- Custom Scripts: `docs/custom-scripts.md`
+- Installation: `docs/installation.md`
+- Troubleshooting: `docs/troubleshooting.md`

docs/custom-scripts.md

@@ -0,0 +1,184 @@
+# Custom Scripts
+
+Shelfmark can run an executable you provide after a download task completes successfully. The script runs after the selected output has finished (for example: transfer to the folder destination, or upload to Booklore).
+
+
+## Quick Start (Recommended)
+
+1. Put your script on the machine that runs Shelfmark.
+1. Make it executable.
+1. Set it in Shelfmark (Settings -> Advanced -> Custom Script Path).
+
+Example:
+
+```bash
+chmod +x /path/to/your/scripts/post_process.sh
+```
+
+### Docker Users
+
+If you run Shelfmark in Docker, the script must exist inside the container. The easiest way is to mount a folder of scripts, then point Shelfmark at the container path in the UI.
+
+```yaml
+services:
+  shelfmark:
+    image: ghcr.io/calibrain/shelfmark:latest
+    volumes:
+      - /path/to/your/scripts:/scripts:ro
+```
+
+Then set:
+
+- Settings -> Advanced -> Custom Script Path: `/scripts/post_process.sh`
+
+<details>
+<summary>Docker Compose: Configure Via Environment Variables (Optional)</summary>
+
+```yaml
+services:
+  shelfmark:
+    environment:
+      - CUSTOM_SCRIPT=/scripts/post_process.sh
+      - CUSTOM_SCRIPT_PATH_MODE=absolute
+      - CUSTOM_SCRIPT_JSON_PAYLOAD=true
+```
+
+</details>
+
+## Script Behaviour
+
+When enabled, Shelfmark runs your script once per successful task:
+
+```bash
+<custom_script_path> "<target_path>"
+```
+
+- `$1` is always set to the target path.
+- If **Custom Script JSON Payload** is enabled, Shelfmark writes a JSON document to stdin (UTF-8).
+- If JSON payload is disabled, stdin is empty (EOF).
+- Timeout: 300 seconds (5 minutes)
+- Exit code: `0` = success; anything else = the task is marked as **Error**
+- Concurrency: downloads can run in parallel, so your script may be invoked concurrently for different tasks.
+- Runtime: the script runs inside the Shelfmark container (if you use Docker) under the same user as Shelfmark.
+
+## The Target Path (`$1`)
+
+Shelfmark chooses a "best single path" for the task:
+
+- If the output produced exactly one local file: that file path.
+- If the output produced multiple local files: a directory path (the common parent directory of those files).
+
+What the target path refers to depends on the output mode:
+
+- Folder output (`output.mode=folder`, `phase=post_transfer`): the final imported file or folder inside your destination.
+- Booklore output (`output.mode=booklore`, `phase=post_upload`): the local file or folder that was uploaded (the destination is remote).
+
+By default, `$1` is an absolute path inside the Shelfmark container (or on your host, if you are not using Docker).
+
+## JSON Payload (stdin)
+
+Configure in: Settings -> Advanced -> Custom Script JSON Payload
+
+When enabled, Shelfmark sends a versioned JSON payload to your script via stdin (and still passes `$1`). This is the recommended way to write robust scripts, especially for multi-file imports (audiobooks) and output-specific context (like Booklore).
+
+- The JSON payload always includes absolute paths in `paths.*`, even if you set Custom Script Path Mode to `relative` for `$1`.
+- `output.mode` tells you which output ran.
+- `output.details` is output-specific. For Booklore output, `output.details.booklore` includes connection details such as `base_url`, `library_id`, and `path_id`.
+- `phase` indicates when the script is running. Current values: `post_transfer` (folder output), `post_upload` (Booklore output).
+- `transfer` is only included for outputs that do a local transfer (for example the folder output).
+
+If JSON payload is disabled, stdin is empty (EOF). Don't `cat` stdin unless you've enabled the payload.
+
+Example payload shape:
+
+```json
+{
+  "version": 1,
+  "phase": "post_transfer",
+  "task": {
+    "task_id": "abc123",
+    "source": "direct",
+    "title": "Foundation",
+    "author": "Isaac Asimov"
+  },
+  "output": {
+    "mode": "folder",
+    "organization_mode": "organize"
+  },
+  "paths": {
+    "destination": "/data/library/books",
+    "target": "/data/library/books/Isaac Asimov/Foundation/Foundation.epub",
+    "final_paths": [
+      "/data/library/books/Isaac Asimov/Foundation/Foundation.epub"
+    ]
+  },
+  "transfer": {
+    "op_counts": {"copy": 1, "move": 0, "hardlink": 0},
+    "use_hardlink": false,
+    "is_torrent": false,
+    "preserve_source": false
+  }
+}
+```
+
+Example (bash + jq) (JSON payload must be enabled):
+
+```bash
+payload="$(cat)"
+mode="$(echo "$payload" | jq -r '.output.mode')"
+title="$(echo "$payload" | jq -r '.task.title')"
+final_paths="$(echo "$payload" | jq -r '.paths.final_paths[]')"
+echo "mode=$mode title=$title" >&2
+echo "$final_paths" >&2
+```
+
+Example (Python) (works whether JSON payload is enabled or not):
+
+```python
+#!/usr/bin/env python3
+import json
+import sys
+
+target = sys.argv[1]
+raw = sys.stdin.read()
+payload = json.loads(raw) if raw.strip() else None
+
+print(f"target={target}", file=sys.stderr)
+if payload:
+    print(f"mode={payload['output']['mode']} phase={payload['phase']}", file=sys.stderr)
+```
+
+<details>
+<summary>Advanced Options</summary>
+
+### Absolute vs Relative Target Paths
+
+Configure in: Settings -> Advanced -> Custom Script Path Mode
+
+This setting controls what gets passed as `$1`:
+
+- `absolute` (default): pass an absolute path.
+- `relative`: pass a path relative to the output's "destination root", and run the script with `$PWD` set to that root.
+
+For folder output, the destination root is your configured destination folder. For Booklore output, it's the local upload folder.
+
+Example (folder destination is `/data/library/books`, and the imported file ended up in `Isaac Asimov/Foundation/Foundation.epub`):
+
+```bash
+# Absolute mode:
+$PWD is unchanged
+$1 = /data/library/books/Isaac Asimov/Foundation/Foundation.epub
+
+# Relative mode:
+$PWD = /data/library/books
+$1 = Isaac Asimov/Foundation/Foundation.epub
+```
+
+Note: if the target is the destination folder itself, `relative` mode may pass `.`.
+
+</details>
+
+## Notes And Caveats
+
+- **Hardlinks and torrents:** if you use hardlinking to keep seeding, avoid scripts that modify file contents, since hardlinked files share data with the seeding copy.
+- **Booklore output mode:** scripts run after upload. `$1` will point at the local uploaded file (or staging folder).

docs/dev/index.md

@@ -0,0 +1,3 @@
+# Developer Documentation
+
+TODO

docs/dev/plugin-settings.md

@@ -16,7 +16,7 @@ The settings system uses a decorator-based registration pattern. Plugins registe
 Add settings to your plugin in 3 steps:
 
 ```python
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     TextField,
     PasswordField,
@@ -190,6 +190,31 @@ HeadingField(
 )
 ```
 
+### CustomComponentField
+
+Render a frontend-registered custom settings component while still using the
+decorator-based schema.
+
+```python
+from shelfmark.core.settings_registry import CustomComponentField
+
+CustomComponentField(
+    key="request_policy_editor",
+    component="request_policy_grid",  # frontend registry key
+    label="Request Policy Rules",
+    description="Custom editor for policy defaults and matrix rules.",
+    value_fields=[
+        SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
+        SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
+        TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
+    ],
+    wrap_in_field_wrapper=True,  # use standard FieldWrapper label/description layout
+)
+```
+
+When `value_fields` is provided, those backing fields are included in
+serialization/save/validation automatically and are hidden from the default renderer.
+
 ## Common Field Properties
 
 All field types support these common properties:
@@ -206,6 +231,7 @@ All field types support these common properties:
 | `requires_restart` | `bool` | `False` | Whether changes require container restart |
 | `show_when` | `dict` | `None` | Conditional visibility (see below) |
 | `disabled_when` | `dict` | `None` | Conditional disable (see below) |
+| `hidden_in_ui` | `bool` | `False` | Hide from default renderer but keep in schema/save path |
 
 ## Conditional Visibility
 
@@ -249,7 +275,7 @@ The field will be greyed out with the specified reason when the condition is met
 Register a group to organize related settings tabs in the sidebar:
 
 ```python
-from cwa_book_downloader.core.settings_registry import register_group
+from shelfmark.core.settings_registry import register_group
 
 # Register a group (do this once, usually in a central config file)
 register_group(
@@ -289,7 +315,7 @@ The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdire
 Use the `config` singleton to read setting values in your plugin code:
 
 ```python
-from cwa_book_downloader.core.config import config
+from shelfmark.core.config import config
 
 # Get a setting value with default fallback
 api_key = config.get("MY_PLUGIN_API_KEY", "")
@@ -312,13 +338,13 @@ The config singleton:
 Here's a complete example for a metadata provider plugin:
 
 ```python
-# cwa_book_downloader/metadata_providers/my_provider.py
+# shelfmark/metadata_providers/my_provider.py
 
-from cwa_book_downloader.metadata_providers.base import (
+from shelfmark.metadata_providers.base import (
     MetadataProvider,
     register_provider,
 )
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -326,7 +352,7 @@ from cwa_book_downloader.core.settings_registry import (
     CheckboxField,
     ActionButton,
 )
-from cwa_book_downloader.core.config import config
+from shelfmark.core.config import config
 
 
 def _test_connection():
@@ -422,15 +448,15 @@ class MyProvider(MetadataProvider):
 Here's a complete example for a release source plugin:
 
 ```python
-# cwa_book_downloader/release_sources/my_source.py
+# shelfmark/release_sources/my_source.py
 
-from cwa_book_downloader.release_sources.base import (
+from shelfmark.release_sources.base import (
     ReleaseSource,
     DownloadHandler,
     register_source,
     register_handler,
 )
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -439,7 +465,7 @@ from cwa_book_downloader.core.settings_registry import (
     SelectField,
     ActionButton,
 )
-from cwa_book_downloader.core.config import config
+from shelfmark.core.config import config
 
 
 def _test_source():

docs/dev/release-sources-plugin-guide.md

@@ -1,6 +1,6 @@
 # Release Sources Plugin Development Guide
 
-This guide explains how to create custom release source plugins for the CWA Book Downloader. The plugin system allows you to add new sources for searching and downloading books while integrating seamlessly with the existing queue, progress reporting, and settings infrastructure.
+This guide explains how to create custom release source plugins for the Shelfmark. The plugin system allows you to add new sources for searching and downloading books while integrating seamlessly with the existing queue, progress reporting, and settings infrastructure.
 
 ## Table of Contents
 
@@ -43,7 +43,7 @@ The release sources system is built around two core interfaces:
 - **DownloadHandler**: Executes the actual download with progress reporting
 
 This separation allows:
-- Different search sources (Anna's Archive, Prowlarr, IRC, etc.)
+- Different search sources (Direct Download, Prowlarr, IRC, etc.)
 - Different download protocols (HTTP, torrent, usenet, etc.)
 - Shared queue and progress infrastructure
 
@@ -77,21 +77,21 @@ class MyHandler(DownloadHandler): ...
 
 ## Quick Start
 
-Create a new file at `cwa_book_downloader/release_sources/my_source.py`:
+Create a new file at `shelfmark/release_sources/my_source.py`:
 
 ```python
 from typing import Callable, List, Optional
 from threading import Event
 
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     Release,
     ReleaseSource,
     DownloadHandler,
     register_source,
     register_handler,
 )
-from cwa_book_downloader.metadata_providers import BookMetadata
-from cwa_book_downloader.core.models import DownloadTask
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.core.models import DownloadTask
 
 
 @register_source("my_source")
@@ -123,11 +123,11 @@ class MyHandler(DownloadHandler):
         return False  # Cancellation via cancel_flag
 ```
 
-Register the import in `cwa_book_downloader/release_sources/__init__.py`:
+Register the import in `shelfmark/release_sources/__init__.py`:
 
 ```python
 # At the bottom of the file
-from cwa_book_downloader.release_sources import my_source  # noqa: F401, E402
+from shelfmark.release_sources import my_source  # noqa: F401, E402
 ```
 
 ---
@@ -139,8 +139,8 @@ The `ReleaseSource` abstract base class defines the search interface:
 ```python
 from abc import ABC, abstractmethod
 from typing import List
-from cwa_book_downloader.metadata_providers import BookMetadata
-from cwa_book_downloader.release_sources import Release, ReleaseColumnConfig
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.release_sources import Release, ReleaseColumnConfig
 
 class ReleaseSource(ABC):
     """Interface for searching a release source."""
@@ -237,7 +237,7 @@ The `DownloadHandler` abstract base class defines the download interface:
 from abc import ABC, abstractmethod
 from typing import Callable, Optional
 from threading import Event
-from cwa_book_downloader.core.models import DownloadTask
+from shelfmark.core.models import DownloadTask
 
 class DownloadHandler(ABC):
     """Interface for executing downloads from a source."""
@@ -371,7 +371,7 @@ class Release:
     info_url: Optional[str] = None      # Link to tracker/info page
 
     protocol: Optional[str] = None   # "http", "torrent", "usenet"
-    indexer: Optional[str] = None    # Display name: "Anna's Archive", "MyAnonamouse"
+    indexer: Optional[str] = None    # Display name: "Direct Download", "My Indexer"
     seeders: Optional[int] = None    # For torrents
 
     extra: Dict = field(default_factory=dict)  # Source-specific metadata
@@ -458,7 +458,7 @@ class BookMetadata:
 ### Decorators
 
 ```python
-from cwa_book_downloader.release_sources import register_source, register_handler
+from shelfmark.release_sources import register_source, register_handler
 
 @register_source("my_source")
 class MySource(ReleaseSource):
@@ -472,7 +472,7 @@ class MyHandler(DownloadHandler):
 ### Registry Functions
 
 ```python
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     get_source,
     get_handler,
     list_available_sources,
@@ -497,8 +497,8 @@ available = list_available_sources()
 
 ```python
 # At the bottom of __init__.py
-from cwa_book_downloader.release_sources import direct_download  # noqa: F401, E402
-from cwa_book_downloader.release_sources import my_source  # noqa: F401, E402
+from shelfmark.release_sources import direct_download  # noqa: F401, E402
+from shelfmark.release_sources import my_source  # noqa: F401, E402
 ```
 
 The `noqa` comments suppress linter warnings about unused imports.
@@ -510,7 +510,7 @@ The `noqa` comments suppress linter warnings about unused imports.
 Register plugin settings using the settings registry decorator:
 
 ```python
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -621,7 +621,7 @@ def test_my_source_connection():
 ### Reading Settings
 
 ```python
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.core.settings_registry import (
     get_setting_value,
     is_value_from_env,
     load_config_file,
@@ -658,7 +658,7 @@ When a value comes from an ENV var, the UI shows a "locked" badge and the field
 Customize how releases are displayed in the release modal by overriding `get_column_config()`:
 
 ```python
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     ReleaseColumnConfig,
     ColumnSchema,
     ColumnRenderType,
@@ -907,7 +907,7 @@ from pathlib import Path
 from typing import Callable, Dict, List, Optional
 from threading import Event
 
-from cwa_book_downloader.release_sources import (
+from shelfmark.release_sources import (
     Release,
     ReleaseSource,
     DownloadHandler,
@@ -921,10 +921,10 @@ from cwa_book_downloader.release_sources import (
     LeadingCellConfig,
     LeadingCellType,
 )
-from cwa_book_downloader.metadata_providers import BookMetadata
-from cwa_book_downloader.core.models import DownloadTask
-from cwa_book_downloader.core.logger import setup_logger
-from cwa_book_downloader.core.settings_registry import (
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.core.models import DownloadTask
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
     register_settings,
     HeadingField,
     TextField,
@@ -934,7 +934,7 @@ from cwa_book_downloader.core.settings_registry import (
     ActionButton,
     load_config_file,
 )
-from cwa_book_downloader.config.env import INGEST_DIR
+from shelfmark.config.env import INGEST_DIR
 
 logger = setup_logger(__name__)
 
@@ -1421,7 +1421,7 @@ PasswordField(
 Use the project's logger for consistent output:
 
 ```python
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -1437,13 +1437,13 @@ logger.error("Failures")                 # Unrecoverable errors
 
 When creating a new plugin:
 
-1. Create `cwa_book_downloader/release_sources/my_plugin.py`
+1. Create `shelfmark/release_sources/my_plugin.py`
 2. Implement `ReleaseSource` subclass with `@register_source("my_plugin")`
 3. Implement `DownloadHandler` subclass with `@register_handler("my_plugin")`
 4. Add settings with `@register_settings("my_plugin", ...)`
-5. Add import to `cwa_book_downloader/release_sources/__init__.py`:
+5. Add import to `shelfmark/release_sources/__init__.py`:
    ```python
-   from cwa_book_downloader.release_sources import my_plugin  # noqa: F401, E402
+   from shelfmark.release_sources import my_plugin  # noqa: F401, E402
    ```
 6. Test `is_available()` returns `True` when configured
 7. Test search returns valid `Release` objects

docs/index.md

@@ -0,0 +1,3 @@
+# Shelfmark Documentation
+
+TODO

docs/installation.md

@@ -0,0 +1,3 @@
+# Installation
+
+TODO

docs/reverse-proxy.md

@@ -0,0 +1,147 @@
+# Reverse Proxy & Subpath Hosting
+
+Shelfmark can run behind a reverse proxy at the root path (recommended) or under a subpath like `/shelfmark`.
+
+## Root path setup (Recommended)
+
+If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`), leave `URL_BASE` empty. This is the simplest option and avoids extra subpath configuration.
+
+```nginx
+server {
+    listen 443 ssl;
+    server_name shelfmark.example.com;
+
+    location / {
+        proxy_pass http://shelfmark:8084;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
+```
+
+## Subpath setup
+
+Running Shelfmark under a subpath like `/shelfmark` is supported without extra rewrite rules.
+
+### 1. Set the base path in Shelfmark
+
+- **UI**: Settings → Advanced → Base Path → `/shelfmark/`
+- **Environment variable**: `URL_BASE=/shelfmark/`
+
+### 2. Configure your reverse proxy
+
+All Shelfmark paths (UI, API, assets, Socket.IO) are served under the base path. A single location block is enough.
+
+---
+
+### Without Authentication Proxy
+
+**Complete Nginx configuration for subpath deployment:**
+
+```nginx
+location /shelfmark/ {
+    proxy_pass http://shelfmark:8084/shelfmark/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+    proxy_send_timeout 86400;
+    proxy_buffering off;
+}
+```
+
+---
+
+### With Authentication Proxy (Authelia, Authentik, etc.)
+
+Shelfmark supports Proxy Authentication. When enabled, Shelfmark trusts the authenticated user from headers set by your auth proxy.
+
+#### Shelfmark Settings
+
+Configure in Settings → Security:
+
+| Setting | Value |
+|---------|-------|
+| Authentication Method | Proxy Authentication |
+| Proxy Auth User Header | `Remote-User` |
+| Proxy Auth Logout URL | `https://auth.example.com/logout` |
+| Proxy Auth Admin Group Header | `Remote-Groups` |
+| Proxy Auth Admin Group Name | `admins` (or your admin group) |
+
+#### Nginx Configuration with Authelia
+
+This example uses Authelia snippets. Adapt for your auth proxy.
+
+**Authelia auth request snippet** (`/etc/nginx/snippets/authelia-authrequest.conf`):
+
+```nginx
+location /authelia {
+    internal;
+    proxy_pass http://authelia:9091/api/authz/auth-request;
+    proxy_pass_request_body off;
+    proxy_set_header Content-Length "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
+    proxy_set_header X-Original-Method $request_method;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+}
+```
+
+**Authelia location snippet** (`/etc/nginx/snippets/authelia-location.conf`):
+
+```nginx
+auth_request /authelia;
+auth_request_set $target_url $scheme://$http_host$request_uri;
+auth_request_set $user $upstream_http_remote_user;
+auth_request_set $groups $upstream_http_remote_groups;
+auth_request_set $name $upstream_http_remote_name;
+auth_request_set $email $upstream_http_remote_email;
+proxy_set_header Remote-User $user;
+proxy_set_header Remote-Groups $groups;
+proxy_set_header Remote-Name $name;
+proxy_set_header Remote-Email $email;
+error_page 401 =302 https://auth.example.com/?rd=$target_url;
+```
+
+**Complete Nginx configuration with Authelia:**
+
+```nginx
+# Include Authelia auth endpoint in your server block
+include /etc/nginx/snippets/authelia-authrequest.conf;
+
+# Main shelfmark location
+location /shelfmark/ {
+    include /etc/nginx/snippets/authelia-location.conf;
+
+    proxy_pass http://shelfmark:8084/shelfmark/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+    proxy_send_timeout 86400;
+    proxy_buffering off;
+}
+```
+
+---
+
+## Health checks
+
+Health checks work at `/shelfmark/api/health` when using a subpath configuration.

docs/troubleshooting.md

@@ -0,0 +1,3 @@
+# Troubleshooting
+
+TODO

docs/url-search-parameters.md

@@ -61,7 +61,7 @@ Some parameters support multiple values by repeating the parameter:
 
 ### Direct Download Mode (default)
 
-All parameters are used to filter results from Anna's Archive.
+All parameters are used to filter results from the direct download source.
 
 ### Universal Mode
 

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/cwa-book-downloader
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/cwa-bd_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+
+    # Cleanup any existing files or folders in the log directory
+    rm -rf "$LOG_DIR"/*
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,10 +56,16 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
@@ -28,34 +78,57 @@ if [ "$TZ" ]; then
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 fi
 
-# Set UID if not set
-if [ -z "$UID" ]; then
-    UID=1000
+# Determine user ID with proper precedence:
+# 1. PUID (LinuxServer.io standard - recommended)
+# 2. UID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+#
+# Note: $UID is a bash builtin that's always set. We use `printenv` to detect
+# if UID was explicitly set as an environment variable (e.g., via docker-compose).
+if [ -n "$PUID" ]; then
+    RUN_UID="$PUID"
+    echo "Using PUID=$RUN_UID"
+elif printenv UID >/dev/null 2>&1; then
+    RUN_UID="$(printenv UID)"
+    echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)"
+else
+    RUN_UID=1000
+    echo "Using default UID=$RUN_UID"
 fi
 
-# Set GID if not set
-if [ -z "$GID" ]; then
-    GID=100
+# Determine group ID with proper precedence:
+# 1. PGID (LinuxServer.io standard - recommended)
+# 2. GID (legacy, for backward compatibility with existing installs)
+# 3. Default to 1000
+if [ -n "$PGID" ]; then
+    RUN_GID="$PGID"
+    echo "Using PGID=$RUN_GID"
+elif [ -n "$GID" ]; then
+    RUN_GID="$GID"
+    echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)"
+else
+    RUN_GID=1000
+    echo "Using default GID=$RUN_GID"
 fi
 
-if ! getent group "$GID" >/dev/null; then
-    echo "Adding group $GID with name appuser"
-    groupadd -g "$GID" appuser
+if ! getent group "$RUN_GID" >/dev/null; then
+    echo "Adding group $RUN_GID with name appuser"
+    groupadd -g "$RUN_GID" appuser
 fi
 
 # Create user if it doesn't exist
-if ! id -u "$UID" >/dev/null 2>&1; then
-    echo "Adding user $UID with name appuser"
-    useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser
+if ! id -u "$RUN_UID" >/dev/null 2>&1; then
+    echo "Adding user $RUN_UID with name appuser"
+    useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser
 fi
 
 # Get username for the UID (whether we just created it or it existed)
-USERNAME=$(getent passwd "$UID" | cut -d: -f1)
-echo "Username for UID $UID is $USERNAME"
+USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1)
+echo "Username for UID $RUN_UID is $USERNAME"
 
 test_write() {
     folder=$1
-    test_file=$folder/calibre-web-automated-book-downloader_TEST_WRITE
+    test_file=$folder/shelfmark_TEST_WRITE
     mkdir -p $folder
     (
         echo 0123456789_TEST | sudo -E -u "$USERNAME" HOME=/app tee $test_file > /dev/null
@@ -75,6 +148,7 @@ test_write() {
 
 make_writable() {
     folder=$1
+    did_full_chown=0
     set +e
     test_write $folder
     is_writable=$?
@@ -84,32 +158,115 @@ make_writable() {
     else
         echo "Folder $folder is not writable, changing ownership"
         change_ownership $folder
-        chmod g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        chmod -R g+r,g+w $folder || echo "Failed to change group permissions for ${folder}, continuing..."
+        did_full_chown=1
+    fi
+    # Fix any misowned subdirectories/files (e.g., from previous runs as root)
+    if [ "$did_full_chown" -eq 0 ] && [ -d "$folder" ]; then
+        echo "Checking for misowned files/directories in $folder"
+        # Stay on the same filesystem to avoid traversing mounted subpaths
+        # (for example read-only bind mounts under /app in dev setups).
+        find "$folder" -xdev -mindepth 1 \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+            -exec chown "$RUN_UID:$RUN_GID" {} + 2>/dev/null || true
     fi
     test_write $folder || echo "Failed to test write to ${folder}, continuing..."
 }
 
+fix_misowned() {
+    folder=$1
+    mkdir -p $folder
+    echo "Checking for misowned files/directories in $folder"
+    # Stay on the same filesystem to avoid traversing mounted subpaths
+    # (for example read-only bind mounts under /app in dev setups).
+    find "$folder" -xdev \( ! -user "$RUN_UID" -o ! -group "$RUN_GID" \) \
+        -exec chown "$RUN_UID:$RUN_GID" {} + 2>/dev/null || true
+}
+
 # Ensure proper ownership of application directories
 change_ownership() {
   folder=$1
   mkdir -p $folder
-  echo "Changing ownership of $folder to $USERNAME:$GID"
-  chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..."
-  chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..."
+  echo "Changing ownership of $folder to $USERNAME:$RUN_GID"
+  chown -R "${RUN_UID}:${RUN_GID}" "${folder}" || echo "Failed to change ownership for ${folder}, continuing..."
 }
 
-change_ownership /app
-change_ownership /var/log/cwa-book-downloader
-change_ownership /tmp/cwa-book-downloader
+fix_misowned /app
+fix_misowned /var/log/shelfmark
+fix_misowned /tmp/shelfmark
+
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
+
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
+fi
 
 # Test write to all folders
-make_writable /cwa-book-ingest
+make_writable ${CONFIG_DIR:-/config}
+make_writable ${INGEST_DIR:-/books}
+
+# Fix permissions on directories configured in settings
+echo "Checking for additional configured directories..."
+if [ -f /app/scripts/fix_permissions.py ]; then
+    configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
+    if [ -n "$configured_dirs" ]; then
+        echo "$configured_dirs" | while read -r dir; do
+            if [ -n "$dir" ] && [ -d "$dir" ]; then
+                echo "Checking configured directory: $dir"
+                make_writable "$dir"
+            fi
+        done
+    fi
+fi
+
+# Fallback to root if config dir is still not writable (common on NAS/Unraid after upgrade from v0.4.0)
+CONFIG_PATH=${CONFIG_DIR:-/config}
+set +e
+test_write "$CONFIG_PATH" >/dev/null 2>&1
+config_ok=$?
+set -e
+
+if [ $config_ok -ne 0 ] && [ "$RUN_UID" != "0" ]; then
+    config_owner=$(stat -c '%u' "$CONFIG_PATH" 2>/dev/null || echo "unknown")
+    if [ "$config_owner" = "0" ]; then
+        echo ""
+        echo "========================================================"
+        echo "WARNING: Permission issue detected!"
+        echo ""
+        echo "Config directory is owned by root but PUID=$RUN_UID."
+        echo "This typically happens after upgrading from v0.4.0 where"
+        echo "PUID/PGID settings were not respected."
+        echo ""
+        echo "Falling back to running as root to prevent data loss."
+        echo ""
+        echo "To fix this permanently, run on your HOST machine:"
+        echo "  chown -R $RUN_UID:$RUN_GID /path/to/config"
+        echo ""
+        echo "Then restart the container."
+        echo "========================================================"
+        echo ""
+        RUN_UID=0
+        RUN_GID=0
+        USERNAME=root
+    fi
+fi
 
 # Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket
 # upgrades work reliably on customer machines.
 # Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase).
 gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]')
-command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} cwa_book_downloader.main:app"
+command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} shelfmark.main:app"
 
 # If DEBUG and not using an external bypass
 if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
@@ -165,18 +322,24 @@ if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
     echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
 fi
 
-# Hacky way to verify /tmp has at least 1MB of space and is writable/readable
+# Verify /tmp has at least 1MB of space and is writable/readable
 echo "Verifying /tmp has enough space"
-rm -f /tmp/test.cwa-bd
-for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd
-sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))")
-[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1)
-rm /tmp/test.cwa-bd
+rm -f /tmp/test.shelfmark
+if dd if=/dev/zero of=/tmp/test.shelfmark bs=1M count=1 2>/dev/null && \
+   [ "$(wc -c < /tmp/test.shelfmark)" -eq 1048576 ]; then
+    rm -f /tmp/test.shelfmark
+    echo "Success: /tmp is writable and readable"
+else
+    echo "Failure: /tmp is not writable or has insufficient space"
+    exit 1
+fi
 
 echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
+# Set umask for file permissions (default: 0022 = files 644, dirs 755)
+UMASK_VALUE=${UMASK:-0022}
+echo "Setting umask to $UMASK_VALUE"
+umask $UMASK_VALUE
 
+stop_file_logging
 exec sudo -E -u "$USERNAME" HOME=/app $command

genDebug.sh

@@ -2,8 +2,8 @@
 
 # Set up log paths
 LOG_ROOT=${LOG_ROOT:-"/var/log"}
-LOG_DIR="$LOG_ROOT/cwa-book-downloader"
-OUTPUT_FILE_NAME="cwa-book-downloader-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
+LOG_DIR="$LOG_ROOT/shelfmark"
+OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
 OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
 
 # Create LOG_DIR if it doesn't exist
@@ -100,14 +100,14 @@ echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
 ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt" 2>&1
+echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
+ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt" 2>&1
+echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
-echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt" 2>&1
+echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 
 # Check Iptables (NAT)

node_modules/.bin/baseline-browser-mapping

@@ -0,0 +1 @@
+../baseline-browser-mapping/dist/cli.js

node_modules/.package-lock.json

@@ -0,0 +1,17 @@
+{
+  "name": "shelfmark",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "node_modules/baseline-browser-mapping": {
+      "version": "2.9.19",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz",
+      "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "baseline-browser-mapping": "dist/cli.js"
+      }
+    }
+  }
+}

node_modules/baseline-browser-mapping/LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

node_modules/baseline-browser-mapping/README.md

@@ -0,0 +1,463 @@
+# [`baseline-browser-mapping`](https://github.com/web-platform-dx/web-features/packages/baseline-browser-mapping)
+
+By the [W3C WebDX Community Group](https://www.w3.org/community/webdx/) and contributors.
+
+`baseline-browser-mapping` provides:
+
+- An `Array` of browsers compatible with Baseline Widely available and Baseline year feature sets via the [`getCompatibleVersions()` function](#get-baseline-widely-available-browser-versions-or-baseline-year-browser-versions).
+- An `Array`, `Object` or `CSV` as a string describing the Baseline feature set support of all browser versions included in the module's data set via the [`getAllVersions()` function](#get-data-for-all-browser-versions).
+
+You can use `baseline-browser-mapping` to help you determine minimum browser version support for your chosen Baseline feature set; or to analyse the level of support for different Baseline feature sets in your site's traffic by joining the data with your analytics data.
+
+## Install for local development
+
+To install the package, run:
+
+`npm install --save-dev baseline-browser-mapping`
+
+`baseline-browser-mapping` depends on `web-features` and `@mdn/browser-compat-data` for core browser version selection, but the data is pre-packaged and minified. This package checks for updates to those modules and the supported [downstream browsers](#downstream-browsers) on a daily basis and is updated frequently. Consider adding a script to your `package.json` to update `baseline-browser-mapping` and using it as part of your build process to ensure your data is as up to date as possible:
+
+```javascript
+"scripts": [
+  "refresh-baseline-browser-mapping": "npm i --save-dev baseline-browser-mapping@latest"
+]
+```
+
+The minimum supported NodeJS version for `baseline-browser-mapping` is v8 in alignment with `browserslist`. For NodeJS versions earlier than v13.2, the [`require('baseline-browser-mapping')`](https://nodejs.org/api/modules.html#requireid) syntax should be used to import the module.
+
+## Keeping `baseline-browser-mapping` up to date
+
+If you are only using this module to generate minimum browser versions for Baseline Widely available or Baseline year feature sets, you don't need to update this module frequently, as the backward looking data is reasonably stable.
+
+However, if you are targeting Newly available, using the [`getAllVersions()`](#get-data-for-all-browser-versions) function or heavily relying on the data for downstream browsers, you should update this module more frequently. If you target a feature cut off date within the last two months and your installed version of `baseline-browser-mapping` has data that is more than 2 months old, you will receive a console warning advising you to update to the latest version when you call `getCompatibleVersions()` or `getAllVersions()`.
+
+If you want to suppress these warnings you can use the `suppressWarnings: true` option in the configuration object passed to `getCompatibleVersions()` or `getAllVersions()`. Alternatively, you can use the `BASELINE_BROWSER_MAPPING_IGNORE_OLD_DATA=true` environment variable when running your build process. This module also respects the `BROWSERSLIST_IGNORE_OLD_DATA=true` environment variable. Environment variables can also be provided in a `.env` file from Node 20 onwards; however, this module does not load .env files automatically to avoid conflicts with other libraries with different requirements. You will need to use `process.loadEnvFile()` or a library like `dotenv` to load .env files before `baseline-browser-mapping` is called.
+
+If you want to ensure [reproducible builds](https://www.wikiwand.com/en/articles/Reproducible_builds), we strongly recommend using the `widelyAvailableOnDate` option to fix the Widely available date on a per build basis to ensure dependent tools provide the same output and you do not produce data staleness warnings. If you are using [`browserslist`](https://github.com/browserslist/browserslist) to target Baseline Widely available, consider automatically updating your `browserslist` configuration in `package.json` or `.browserslistrc` to `baseline widely available on {YYYY-MM-DD}` as part of your build process to ensure the same or sufficiently similar list of minimum browsers is reproduced for historical builds.
+
+## Importing `baseline-browser-mapping`
+
+This module exposes two functions: `getCompatibleVersions()` and `getAllVersions()`, both which can be imported directly from `baseline-browser-mapping`:
+
+```javascript
+import {
+  getCompatibleVersions,
+  getAllVersions,
+} from "baseline-browser-mapping";
+```
+
+If you want to load the script and data directly in a web page without hosting it yourself, consider using a CDN:
+
+```html
+<script type="module">
+  import {
+    getCompatibleVersions,
+    getAllVersions,
+  } from "https://cdn.jsdelivr.net/npm/baseline-browser-mapping";
+</script>
+```
+
+## Get Baseline Widely available browser versions or Baseline year browser versions
+
+To get the current list of minimum browser versions compatible with Baseline Widely available features from the core browser set, call the `getCompatibleVersions()` function:
+
+```javascript
+getCompatibleVersions();
+```
+
+Executed on 7th March 2025, the above code returns the following browser versions:
+
+```javascript
+[
+  { browser: "chrome", version: "105", release_date: "2022-09-02" },
+  {
+    browser: "chrome_android",
+    version: "105",
+    release_date: "2022-09-02",
+  },
+  { browser: "edge", version: "105", release_date: "2022-09-02" },
+  { browser: "firefox", version: "104", release_date: "2022-08-23" },
+  {
+    browser: "firefox_android",
+    version: "104",
+    release_date: "2022-08-23",
+  },
+  { browser: "safari", version: "15.6", release_date: "2022-09-02" },
+  {
+    browser: "safari_ios",
+    version: "15.6",
+    release_date: "2022-09-02",
+  },
+];
+```
+
+> [!NOTE]
+> The minimum versions of each browser are not necessarily the final release before the Widely available cutoff date of `TODAY - 30 MONTHS`. Some earlier versions will have supported the full Widely available feature set.
+
+### `getCompatibleVersions()` configuration options
+
+`getCompatibleVersions()` accepts an `Object` as an argument with configuration options. The defaults are as follows:
+
+```javascript
+{
+  targetYear: undefined,
+  widelyAvailableOnDate: undefined,
+  includeDownstreamBrowsers: false,
+  listAllCompatibleVersions: false,
+  suppressWarnings: false
+}
+```
+
+#### `targetYear`
+
+The `targetYear` option returns the minimum browser versions compatible with all **Baseline Newly available** features at the end of the specified calendar year. For example, calling:
+
+```javascript
+getCompatibleVersions({
+  targetYear: 2020,
+});
+```
+
+Returns the following versions:
+
+```javascript
+[
+  { browser: "chrome", version: "87", release_date: "2020-11-19" },
+  {
+    browser: "chrome_android",
+    version: "87",
+    release_date: "2020-11-19",
+  },
+  { browser: "edge", version: "87", release_date: "2020-11-19" },
+  { browser: "firefox", version: "83", release_date: "2020-11-17" },
+  {
+    browser: "firefox_android",
+    version: "83",
+    release_date: "2020-11-17",
+  },
+  { browser: "safari", version: "14", release_date: "2020-09-16" },
+  { browser: "safari_ios", version: "14", release_date: "2020-09-16" },
+];
+```
+
+> [!NOTE]
+> The minimum version of each browser is not necessarily the final version released in that calendar year. In the above example, Firefox 84 was the final version released in 2020; however Firefox 83 supported all of the features that were interoperable at the end of 2020.
+> [!WARNING]
+> You cannot use `targetYear` and `widelyAavailableDate` together. Please only use one of these options at a time.
+
+#### `widelyAvailableOnDate`
+
+The `widelyAvailableOnDate` option returns the minimum versions compatible with Baseline Widely available on a specified date in the format `YYYY-MM-DD`:
+
+```javascript
+getCompatibleVersions({
+  widelyAvailableOnDate: `2023-04-05`,
+});
+```
+
+> [!TIP]
+> This option is useful if you provide a versioned library that targets Baseline Widely available on each version's release date and you need to provide a statement on minimum supported browser versions in your documentation.
+
+#### `includeDownstreamBrowsers`
+
+Setting `includeDownstreamBrowsers` to `true` will include browsers outside of the Baseline core browser set where it is possible to map those browsers to an upstream Chromium or Gecko version:
+
+```javascript
+getCompatibleVersions({
+  includeDownstreamBrowsers: true,
+});
+```
+
+For more information on downstream browsers, see [the section on downstream browsers](#downstream-browsers) below.
+
+#### `includeKaiOS`
+
+KaiOS is an operating system and app framework based on the Gecko engine from Firefox. KaiOS is based on the Gecko engine and feature support can be derived from the upstream Gecko version that each KaiOS version implements. However KaiOS requires other considerations beyond feature compatibility to ensure a good user experience as it runs on device types that do not have either mouse and keyboard or touch screen input in the way that all the other browsers supported by this module do.
+
+```javascript
+getCompatibleVersions({
+  includeDownstreamBrowsers: true,
+  includeKaiOS: true,
+});
+```
+
+> [!NOTE]
+> Including KaiOS requires you to include all downstream browsers using the `includeDownstreamBrowsers` option.
+
+#### `listAllCompatibleVersions`
+
+Setting `listAllCompatibleVersions` to true will include the minimum versions of each compatible browser, and all the subsequent versions:
+
+```javascript
+getCompatibleVersions({
+  listAllCompatibleVersions: true,
+});
+```
+
+#### `suppressWarnings`
+
+Setting `suppressWarnings` to `true` will suppress the console warning about old data:
+
+```javascript
+getCompatibleVersions({
+  suppressWarnings: true,
+});
+```
+
+## Get data for all browser versions
+
+You may want to obtain data on all the browser versions available in this module for use in an analytics solution or dashboard. To get details of each browser version's level of Baseline support, call the `getAllVersions()` function:
+
+```javascript
+import { getAllVersions } from "baseline-browser-mapping";
+
+getAllVersions();
+```
+
+By default, this function returns an `Array` of `Objects` and excludes downstream browsers:
+
+```javascript
+[
+  ...
+  {
+    browser: "firefox_android", // Browser name
+    version: "125", // Browser version
+    release_date: "2024-04-16", // Release date
+    year: 2023, // Baseline year feature set the version supports
+    wa_compatible: true // Whether the browser version supports Widely available
+  },
+  ...
+]
+```
+
+For browser versions in `@mdn/browser-compat-data` that were released before Baseline can be defined, i.e. Baseline 2015, the `year` property is always the string: `"pre_baseline"`.
+
+### Understanding which browsers support Newly available features
+
+You may want to understand which recent browser versions support all Newly available features. You can replace the `wa_compatible` property with a `supports` property using the `useSupport` option:
+
+```javascript
+getAllVersions({
+  useSupports: true,
+});
+```
+
+The `supports` property is optional and has two possible values:
+
+- `widely` for browser versions that support all Widely available features.
+- `newly` for browser versions that support all Newly available features.
+
+Browser versions that do not support Widely or Newly available will not include the `support` property in the `array` or `object` outputs, and in the CSV output, the `support` column will contain an empty string. Browser versions that support all Newly available features also support all Widely available features.
+
+### `getAllVersions()` Configuration options
+
+`getAllVersions()` accepts an `Object` as an argument with configuration options. The defaults are as follows:
+
+```javascript
+{
+  includeDownstreamBrowsers: false,
+  outputFormat: "array",
+  suppressWarnings: false
+}
+```
+
+#### `includeDownstreamBrowsers` (in `getAllVersions()` output)
+
+As with `getCompatibleVersions()`, you can set `includeDownstreamBrowsers` to `true` to include the Chromium and Gecko downstream browsers [listed below](#list-of-downstream-browsers).
+
+```javascript
+getAllVersions({
+  includeDownstreamBrowsers: true,
+});
+```
+
+Downstream browsers include the same properties as core browsers, as well as the `engine`they use and `engine_version`, for example:
+
+```javascript
+[
+  ...
+  {
+    browser: "samsunginternet_android",
+    version: "27.0",
+    release_date: "2024-11-06",
+    engine: "Blink",
+    engine_version: "125",
+    year: 2023,
+    supports: "widely"
+  },
+  ...
+]
+```
+
+#### `includeKaiOS` (in `getAllVersions()` output)
+
+As with `getCompatibleVersions()` you can include KaiOS in your output. The same requirement to have `includeDownstreamBrowsers: true` applies.
+
+```javascript
+getAllVersions({
+  includeDownstreamBrowsers: true,
+  includeKaiOS: true,
+});
+```
+
+#### `suppressWarnings` (in `getAllVersions()` output)
+
+As with `getCompatibleVersions()`, you can set `suppressWarnings` to `true` to suppress the console warning about old data:
+
+```javascript
+getAllVersions({
+  suppressWarnings: true,
+});
+```
+
+#### `outputFormat`
+
+By default, this function returns an `Array` of `Objects` which can be manipulated in Javascript or output to JSON.
+
+To return an `Object` that nests keys , set `outputFormat` to `object`:
+
+```javascript
+getAllVersions({
+  outputFormat: "object",
+});
+```
+
+In thise case, `getAllVersions()` returns a nested object with the browser [IDs listed below](#list-of-downstream-browsers) as keys, and versions as keys within them:
+
+```javascript
+{
+  "chrome": {
+    "53": {
+      "year": 2016,
+      "release_date": "2016-09-07"
+    },
+    ...
+}
+```
+
+Downstream browsers will include extra fields for `engine` and `engine_versions`
+
+```javascript
+{
+  ...
+  "webview_android": {
+    "53": {
+      "year": 2016,
+      "release_date": "2016-09-07",
+      "engine": "Blink",
+      "engine_version": "53"
+    },
+  ...
+}
+```
+
+To return a `String` in CSV format, set `outputFormat` to `csv`:
+
+```javascript
+getAllVersions({
+  outputFormat: "csv",
+});
+```
+
+`getAllVersions` returns a `String` with a header row and comma-separated values for each browser version that you can write to a file or pass to another service. Core browsers will have "NULL" as the value for their `engine` and `engine_version`:
+
+```csv
+"browser","version","year","supports","release_date","engine","engine_version"
+...
+"chrome","24","pre_baseline","","2013-01-10","NULL","NULL"
+...
+"chrome","53","2016","","2016-09-07","NULL","NULL"
+...
+"firefox","135","2024","widely","2025-02-04","NULL","NULL"
+"firefox","136","2024","newly","2025-03-04","NULL","NULL"
+...
+"ya_android","20.12","2020","year_only","2020-12-20","Blink","87"
+...
+```
+
+> [!NOTE]
+> The above example uses `"includeDownstreamBrowsers": true`
+
+### Static resources
+
+The outputs of `getAllVersions()` are available as JSON or CSV files generated on a daily basis and hosted on GitHub pages:
+
+- Core browsers only
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_array.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_object.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions.csv)
+- Core browsers only, with `supports` property
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_array_with_supports.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_object_with_supports.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/all_versions_with_supports.csv)
+- Including downstream browsers
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_array.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_object.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions.csv)
+- Including downstream browsers with `supports` property
+  - [Array](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_array_with_supports.json)
+  - [Object](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_object_with_supports.json)
+  - [CSV](https://web-platform-dx.github.io/baseline-browser-mapping/with_downstream/all_versions_with_supports.csv)
+
+These files are updated on a daily basis.
+
+## CLI
+
+`baseline-browser-mapping` includes a command line interface that exposes the same data and options as the `getCompatibleVersions()` function. To learn more about using the CLI, run:
+
+```sh
+npx baseline-browser-mapping --help
+```
+
+## Downstream browsers
+
+### Limitations
+
+The browser versions in this module come from two different sources:
+
+- MDN's `browser-compat-data` module.
+- Parsed user agent strings provided by [useragents.io](https://useragents.io/)
+
+MDN `browser-compat-data` is an authoritative source of information for the browsers it contains. The release dates for the Baseline core browser set and the mapping of downstream browsers to Chromium versions should be considered accurate.
+
+Browser mappings from useragents.io are provided on a best effort basis. They assume that browser vendors are accurately stating the Chromium version they have implemented. The initial set of version mappings was derived from a bulk export in November 2024. This version was iterated over with a Regex match looking for a major Chrome version and a corresponding version of the browser in question, e.g.:
+
+`Mozilla/5.0 (Linux; U; Android 10; en-US; STK-L21 Build/HUAWEISTK-L21) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/100.0.4896.58 UCBrowser/13.8.2.1324 Mobile Safari/537.36`
+
+Shows UC Browser Mobile 13.8 implementing Chromium 100, and:
+
+`Mozilla/5.0 (Linux; arm_64; Android 11; Redmi Note 8 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.123 YaBrowser/24.10.2.123.00 SA/3 Mobile Safari/537.36`
+
+Shows Yandex Browser Mobile 24.10 implementing Chromium 128. The Chromium version from this string is mapped to the corresponding Chrome version from MDN `browser-compat-data`.
+
+> [!NOTE]
+> Where possible, approximate release dates have been included based on useragents.io "first seen" data. useragents.io does not have "first seen" dates prior to June 2020. However, these browsers' Baseline compatibility is determined by their Chromium or Gecko version, so their release dates are more informative than critical.
+
+This data is updated on a daily basis using a [script](https://github.com/web-platform-dx/web-features/tree/main/scripts/refresh-downstream.ts) triggered by a GitHub [action](https://github.com/web-platform-dx/web-features/tree/main/.github/workflows/refresh_downstream.yml). Useragents.io provides a private API for this module which exposes the last 7 days of newly seen user agents for the currently tracked browsers. If a new major version of one of the tracked browsers is encountered with a Chromium version that meets or exceeds the previous latest version of that browser, it is added to the [src/data/downstream-browsers.json](src/data/downstream-browsers.json) file with the date it was first seen by useragents.io as its release date.
+
+KaiOS is an exception - its upstream version mappings are handled separately from the other browsers because they happen very infrequently.
+
+### List of downstream browsers
+
+| Browser               | ID                        | Core    | Source                    |
+| --------------------- | ------------------------- | ------- | ------------------------- |
+| Chrome                | `chrome`                  | `true`  | MDN `browser-compat-data` |
+| Chrome for Android    | `chrome_android`          | `true`  | MDN `browser-compat-data` |
+| Edge                  | `edge`                    | `true`  | MDN `browser-compat-data` |
+| Firefox               | `firefox`                 | `true`  | MDN `browser-compat-data` |
+| Firefox for Android   | `firefox_android`         | `true`  | MDN `browser-compat-data` |
+| Safari                | `safari`                  | `true`  | MDN `browser-compat-data` |
+| Safari on iOS         | `safari_ios`              | `true`  | MDN `browser-compat-data` |
+| Opera                 | `opera`                   | `false` | MDN `browser-compat-data` |
+| Opera Android         | `opera_android`           | `false` | MDN `browser-compat-data` |
+| Samsung Internet      | `samsunginternet_android` | `false` | MDN `browser-compat-data` |
+| WebView Android       | `webview_android`         | `false` | MDN `browser-compat-data` |
+| QQ Browser Mobile     | `qq_android`              | `false` | useragents.io             |
+| UC Browser Mobile     | `uc_android`              | `false` | useragents.io             |
+| Yandex Browser Mobile | `ya_android`              | `false` | useragents.io             |
+| KaiOS                 | `kai_os`                  | `false` | Manual                    |
+| Facebook for Android  | `facebook_android`        | `false` | useragents.io             |
+| Instagram for Android | `instagram_android`       | `false` | useragents.io             |
+
+> [!NOTE]
+> All the non-core browsers currently included implement Chromium or Gecko. Their inclusion in any of the above methods is based on the Baseline feature set supported by the Chromium or Gecko version they implement, not their release date.

node_modules/baseline-browser-mapping/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "baseline-browser-mapping",
+  "main": "./dist/index.cjs",
+  "version": "2.9.19",
+  "description": "A library for obtaining browser versions with their maximum supported Baseline feature set and Widely Available status.",
+  "exports": {
+    ".": {
+      "require": "./dist/index.cjs",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./legacy": {
+      "require": "./dist/index.cjs",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "jsdelivr": "./dist/index.js",
+  "files": [
+    "dist/*",
+    "!dist/scripts/*",
+    "LICENSE.txt",
+    "README.md"
+  ],
+  "types": "./dist/index.d.ts",
+  "type": "module",
+  "bin": {
+    "baseline-browser-mapping": "dist/cli.js"
+  },
+  "scripts": {
+    "fix-cli-permissions": "output=$(npx baseline-browser-mapping 2>&1); path=$(printf '%s\n' \"$output\" | sed -n 's/^.*: \\(.*\\): Permission denied$/\\1/p; t; s/^\\(.*\\): Permission denied$/\\1/p'); if [ -n \"$path\" ]; then echo \"Permission denied for: $path\"; echo \"Removing $path ...\"; rm -rf \"$path\"; else echo \"$output\"; fi",
+    "test:format": "npx prettier --check .",
+    "test:lint": "npx eslint .",
+    "test:jasmine": "npx jasmine",
+    "test:jasmine-browser": "npx jasmine-browser-runner runSpecs --config ./spec/support/jasmine-browser.js",
+    "test": "npm run build && npm run fix-cli-permissions && npm run test:format && npm run test:lint && npm run test:jasmine && npm run test:jasmine-browser",
+    "build": "rm -rf dist; npx prettier . --write; rollup -c; rm -rf ./dist/scripts/expose-data.d.ts ./dist/cli.d.ts",
+    "refresh-downstream": "npx tsx scripts/refresh-downstream.ts",
+    "refresh-static": "npx tsx scripts/refresh-static.ts",
+    "update-data-file": "npx tsx scripts/update-data-file.ts; npx prettier ./src/data/data.js --write",
+    "update-data-dependencies": "npm i @mdn/browser-compat-data@latest web-features@latest -D",
+    "check-data-changes": "git diff --name-only | grep -q '^src/data/data.js$' && echo 'changes-available=TRUE' || echo 'changes-available=FALSE'"
+  },
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@mdn/browser-compat-data": "^7.2.5",
+    "@rollup/plugin-terser": "^0.4.4",
+    "@rollup/plugin-typescript": "^12.1.3",
+    "@types/node": "^22.15.17",
+    "eslint-plugin-new-with-error": "^5.0.0",
+    "jasmine": "^5.8.0",
+    "jasmine-browser-runner": "^3.0.0",
+    "jasmine-spec-reporter": "^7.0.0",
+    "prettier": "^3.5.3",
+    "rollup": "^4.44.0",
+    "tslib": "^2.8.1",
+    "typescript": "^5.7.2",
+    "typescript-eslint": "^8.35.0",
+    "web-features": "^3.14.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/web-platform-dx/baseline-browser-mapping.git"
+  }
+}

package-lock.json

@@ -0,0 +1,22 @@
+{
+  "name": "shelfmark",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "devDependencies": {
+        "baseline-browser-mapping": "^2.9.19"
+      }
+    },
+    "node_modules/baseline-browser-mapping": {
+      "version": "2.9.19",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz",
+      "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "baseline-browser-mapping": "dist/cli.js"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "baseline-browser-mapping": "^2.9.19"
+  }
+}

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "shelfmark"
+version = "0.1.0"
+description = "Shelfmark - Book Downloader"
+requires-python = ">=3.10"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--tb=short",
+]
+markers = [
+    "integration: marks tests that require running services (deselect with '-m \"not integration\"')",
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "e2e: marks end-to-end tests that require the full application stack",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+ignore_missing_imports = true

readme.md

@@ -1,365 +1,264 @@
-# 📚 Calibre-Web-Automated-Book-Downloader
+# 📚 Shelfmark: Book Downloader
 
-<img src="src/frontend/public/logo.png" alt="Calibre-Web Automated Book Downloader" width="200">
+Formerly *Calibre Web Automated Book Downloader (CWABD)*
 
-An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
+<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
+
+Shelfmark is a unified web interface for searching and aggregating books and audiobook downloads from multiple sources - all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library.
+
+**Fully standalone** - no external dependencies required. Works great alongside the following library tools, with support for automatic imports:
+- [Calibre](https://calibre-ebook.com/)
+- [Calibre-Web](https://github.com/janeczku/calibre-web)
+- [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated)
+- [Booklore](https://github.com/booklore-app/booklore)
+- [Audiobookshelf](https://github.com/advplyr/audiobookshelf)
 
 ## ✨ Features
 
-- 🌐 User-friendly web interface for book search and download
-- 🔄 Automated download to your specified ingest folder
-- 🔌 Seamless integration with Calibre-Web-Automated
-- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr)
-- 🛡️ Cloudflare bypass capability for reliable downloads
-- 🐳 Docker-based deployment for quick setup
+- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
+- **Multiple sources** - Popular archive websites, Torrent, Usenet and IRC download support
+- **Audiobook support** - Full audiobook search and download with dedicated processing
+- **Real-Time Progress** - Unified download queue with live status updates across all sources
+- **Two Search Modes**:
+  - **Direct** - Search popular web sources
+  - **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
+- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
 
 ## 🖼️ Screenshots
 
-![Homescreen](README_images/homescreen.png 'Homescreen')
+**Home screen**
+![Home screen](README_images/homescreen.png 'Home screen')
 
+**Search results**
 ![Search results](README_images/search-results.png 'Search results')
 
-![Active downloads](README_images/downloads.png 'Active downloads')
+**Multi-source downloads**
+![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads')
+
+**Download queue**
+![Download queue](README_images/downloads.png 'Download queue')
 
 ## 🚀 Quick Start
 
 ### Prerequisites
 
-- Docker
-- Docker Compose
-- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended)
+- Docker & Docker Compose
 
-### Installation Steps
-
-1. Get the docker-compose.yml:
+### Installation
 
+1. Download the [docker-compose file](compose/docker-compose.yml):
    ```bash
-   curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml
+   curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
    ```
 
 2. Start the service:
-
    ```bash
    docker compose up -d
    ```
 
-3. Access the web interface at `http://localhost:8084`
+3. Open `http://localhost:8084`
 
-## ⚙️ Configuration
+That's it! Configure settings through the web interface as needed.
 
-### Environment Variables
-
-#### Application Settings
-
-| Variable          | Description             | Default Value      |
-| ----------------- | ----------------------- | ------------------ |
-| `FLASK_PORT`      | Web interface port      | `8084`             |
-| `FLASK_HOST`      | Web interface binding   | `0.0.0.0`          |
-| `DEBUG`           | Debug mode toggle       | `false`            |
-| `INGEST_DIR`      | Book download directory | `/cwa-book-ingest` |
-| `TZ`              | Container timezone      | `UTC`              |
-| `UID`             | Runtime user ID         | `1000`             |
-| `GID`             | Runtime group ID        | `100`              |
-| `CWA_DB_PATH`     | Calibre-Web's database  | None               |
-| `ENABLE_LOGGING`  | Enable log file         | `true`             |
-| `LOG_LEVEL`       | Log level to use        | `info`             |
-| `SESSION_COOKIE_SECURE` | Secure cookie enforcement - Use for HTTPS connections only | `false` |
-| `CALIBRE_WEB_URL` | Custom WebUI library link | None |
-| `BYPASS_WARMUP_ON_CONNECT` | Warm up Cloudflare bypasser when first client connects | `true` |
-
-If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
-
-Set `CALIBRE_WEB_URL` to your Calibre-Web / Booklore base URL. A ‘Go to library’ button will appear in the Web UI for quick access while downloading, and it also provides library access when CWA-BD is installed as a mobile PWA.
-
-If logging is enabled, log folder default location is `/var/log/cwa-book-downloader`
-Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages.
-
-Note that if using TOR, the TZ will be calculated automatically based on IP.
-
-#### Download Settings
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `MAX_RETRY`            | Maximum retry attempts                                    | `3`                               |
-| `DEFAULT_SLEEP`        | Retry delay (seconds)                                     | `5`                               |
-| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds)                           | `5`                               |
-| `SUPPORTED_FORMATS`    | Supported book formats                                    | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
-| `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
-| `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
-| `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
-| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
-| `ALLOW_USE_WELIB`       | Allow usage of welib for downloading books if found there | `true`                            |
-
-If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
-
-Use the following environment variables to set specific folders in which to download 
-different content types (Book, Magazine, Comic, etc.):
-
-| Variable                        | Description                    | Default Value |
-|---------------------------------|--------------------------------|---------------|
-| `INGEST_DIR_BOOK_FICTION`       | Book (fiction) folder name     | ``            |
-| `INGEST_DIR_BOOK_NON_FICTION`   | Book (non-fiction) folder name | ``            |
-| `INGEST_DIR_BOOK_UNKNOWN`       | Book (unknown) folder name     | ``            |
-| `INGEST_DIR_MAGAZINE`           | Magazine folder name           | ``            |
-| `INGEST_DIR_COMIC_BOOK`         | Comic book folder name         | ``            |
-| `INGEST_DIR_AUDIOBOOK`          | Audiobook folder name          | ``            |
-| `INGEST_DIR_STANDARDS_DOCUMENT` | Standards document folder name | ``            |
-| `INGEST_DIR_MUSICAL_SCORE`      | Musical score folder name      | ``            |
-
-If no specific path is set for a content type the default is `INGEST_DIR`.  
-Remember to map the specified paths to where your instance of Calibre-Web-Automated (CWA) will find them, e.g.:  
-```
-volumes:
-    - /tmp/data/calibre-web/comicbook-ingest:/cwa-comicbook-ingest
-```
-if `INGEST_DIR_COMIC_BOOK=/cwa-comicbook-ingest` and your CWA is configured to use `/tmp/data/calibre-web/comicbook-ingest` 
-for comic books.
-
-
-#### AA 
-
-| Variable               | Description                                               | Default Value                     |
-| ---------------------- | --------------------------------------------------------- | --------------------------------- |
-| `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
-
-If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
-If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.
-
-#### Network Settings
-
-| Variable               | Description                     | Default Value           |
-| ---------------------- | ------------------------------- | ----------------------- |
-| `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
-| `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
-| `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | DNS configuration               | `auto`                  |
-| `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
-
-**Proxy Configuration**
-
-For proxy configuration, you can specify URLs in the following format:
-```bash
-# Basic proxy
-HTTP_PROXY=http://proxy.example.com:8080
-HTTPS_PROXY=http://proxy.example.com:8080
-
-# Proxy with authentication
-HTTP_PROXY=http://username:password@proxy.example.com:8080
-HTTPS_PROXY=http://username:password@proxy.example.com:8080
-```
-
-**DNS Configuration**
-
-The `CUSTOM_DNS` setting controls how DNS resolution works. By default, it is set to `auto` which provides automatic failover for reliable connectivity.
-
-**Auto Mode (Default)**
-
-When `CUSTOM_DNS=auto`, the application starts with your system's default DNS. If DNS resolution fails, it automatically rotates through alternative providers using DNS over HTTPS (DoH):
-
-1. System DNS (initial)
-2. Cloudflare (1.1.1.1)
-3. Google (8.8.8.8)
-4. Quad9 (9.9.9.9)
-5. OpenDNS (208.67.222.222)
-
-This automatic rotation helps bypass ISP-level blocks and DNS issues without any manual configuration.
-
-**Manual DNS Configuration**
-
-If you prefer to use a specific DNS configuration, you can override the auto behavior:
-
-1. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS (8.8.8.8, 8.8.4.4)
-   - `quad9` - Quad9 DNS (9.9.9.9, 149.112.112.112)
-   - `cloudflare` - Cloudflare DNS (1.1.1.1, 1.0.0.1)
-   - `opendns` - OpenDNS (208.67.222.222, 208.67.220.220)
-
-2. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
-   - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses
-
-When using preset providers, you can optionally enable DNS over HTTPS with `USE_DOH=true`:
-```bash
-CUSTOM_DNS=cloudflare
-USE_DOH=true
-```
-
-Note: When using custom IP addresses, the `USE_DOH` flag is ignored since DoH requires a known provider endpoint.
-
-#### Custom configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `CUSTOM_SCRIPT`        | Path to an executable script that tuns after each download  | ``                      |
-
-If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation.
-
-The script is called with the full path of the downloaded file as its argument. Important notes:
-- The script must preserve the original filename for proper processing
-- The file can be modified or even deleted if needed
-- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted)
-
-You can specify these configuration in this format :
-```
-environment:
-  - CUSTOM_SCRIPT=/scripts/process-book.sh
-
-volumes:
-  - local/scripts/custom_script.sh:/scripts/process-book.sh
-```
-
-### Volume Configuration
+### Volume Setup
 
 ```yaml
 volumes:
-  - /your/local/path:/cwa-book-ingest
-  - /cwa/config/path/app.db:/auth/app.db:ro
+  - /your/config/path:/config # Config, database, and artwork cache directory
+  - /your/download/path:/books # Downloaded books
+  - /client/path:/client/path # Optional: For Torrent/Usenet downloads, match your client directory exactly. 
 ```
-**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777
 
-Mount should align with your Calibre-Web-Automated ingest folder.
+> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import.
 
-## Variants:
+> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors.
 
-### 🧅 Tor Variant
+## ⚙️ Configuration
 
-This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions.
+### Search Modes
 
-To use the Tor variant:
+**Direct** (default)
+- Works out of the box, no setup required
+- Searches a huge library of books directly
+- Returns downloadable releases immediately
 
-1.  Get the Tor-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.tor.yml up -d
-    ```
+**Universal**
+- Cleaner search results via metadata providers (Hardcover is recommended)
+- Aggregates releases from multiple configured sources
+- Full Audiobook support
+- Requires manual setup (API keys, additional sources)
 
-**Important Considerations for Tor:**
+### Environment Variables
 
-*   **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying.
-*   **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set.
-*   **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor.
+Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
 
-### External Cloudflare resolver variant
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FLASK_PORT` | Web interface port | `8084` |
+| `INGEST_DIR` | Book download directory | `/books` |
+| `TZ` | Container timezone | `UTC` |
+| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` |
+| `SEARCH_MODE` | `direct` or `universal` | `direct` |
+| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
 
-This variant allows the application to use an external service to bypass Cloudflare protection, instead of relying on the built-in bypasser. This is useful if you already have a dedicated Cloudflare resolver (such as [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) or compatible services like [ByParr](https://github.com/ThePhaseless/Byparr)) running elsewhere.
+See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
 
-#### How it works:
+Some of the additional options available in Settings:
+- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
+- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
+- **AudiobookBay** - Web scraping source for audiobook torrents (audiobooks only)
+- **IRC** - Add details for IRC book sources and download directly from the UI
+- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header
+- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
+- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
+- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
+- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
 
-- When enabled, all requests that require Cloudflare bypass are sent to your external resolver service.
-- The application communicates with the resolver using its API.
+## 🐳 Docker Variants
 
-#### Configuration
-
-| Variable               | Description                                                 | Default Value           |
-| ---------------------- | ----------------------------------------------------------- | ----------------------- |
-| `EXT_BYPASSER_URL`     | The full URL of your external resolver (required)           |                         |
-| `EXT_BYPASSER_PATH`    | API path for the resolver (usually `/v1`)                   | `/v1`                   |
-| `EXT_BYPASSER_TIMEOUT` | Timeout for page loading (in milliseconds)                  | `60000`                 |
-
-#### Important
-
-This feature follows the same configuration of the built-in Cloudflare bypasser, so you should turn on the `USE_CF_BYPASS` configuration to enable it.
-
-#### To use the External Cloudflare resolver variant:
-
-1.  Get the extbp-specific docker-compose file:
-    ```bash
-    curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.extbp.yml
-    ```
-2.  Start the service using this file:
-    ```bash
-    docker compose -f docker-compose.extbp.yml up -d
-    ```
-
-#### Compatibility:
-This feature is designed to work with any resolver that implements the `FlareSolverr` API schema, including `ByParr` and similar projects.
-
-#### Internal vs External Bypasser
-
-The **internal bypasser** (default) is custom-designed for this application's specific needs. It handles session management, cookie persistence, and retry logic optimized for book downloading workflows. For most users, this provides the most reliable experience out of the box.
-
-The **external bypasser** is better suited if you:
-- Already run FlareSolverr/ByParr for other services and want to consolidate
-- Need to share bypass infrastructure across multiple applications
-- Want to offload browser automation to a dedicated, more powerful container
-
-If you're unsure which to use, start with the default internal bypasser.
-
-## 🏗️ Architecture
-
-The application consists of a Flask backend with a React-based frontend:
-
-### Backend
-- **Flask Application**: Python-based backend (`app.py`, `backend.py`) providing REST API and WebSocket support
-- **Download Manager**: Handles book search, download requests, and queue management (`downloader.py`, `book_manager.py`)
-- **Network Layer**: Cloudflare bypass and proxy support (`cloudflare_bypasser.py`, `network.py`)
-
-### Frontend
-- **React + TypeScript**: Modern web interface built with Vite (`src/frontend`)
-- **Real-time Updates**: WebSocket integration for live download status
-- **Responsive UI**: TailwindCSS-based design for mobile and desktop
-
-For frontend development, use the provided Makefile:
+### Standard
 ```bash
-make install  # Install dependencies
-make dev      # Start development server
-make build    # Build for production
-```
-If you run the docker compose file, the frontend will be built and served automatically. But if you run the frontend dev server it will supercede the docker compose frontend.
-
-## 🏥 Health Monitoring
-
-Built-in health checks monitor:
-
-- Web interface availability
-- Download service status
-- Cloudflare bypass service connection
-
-Checks run every 30 seconds with a 30-second timeout and 3 retries.
-You can enable by adding this to your compose :
-```
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD curl -s http://localhost:8084/api/status || exit 1
+docker compose up -d
 ```
 
-## 📝 Logging
+The full-featured image with built-in Cloudflare bypass.
 
-Logs are available in:
+#### Enable Tor Routing
+Routes all traffic through Tor for enhanced privacy:
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
+docker compose -f docker-compose.tor.yml up -d
+```
 
-- Container: `/var/logs/cwa-book-downloader.log`
-- Docker logs: Access via `docker logs`
+**Notes:**
+- Requires `NET_ADMIN` and `NET_RAW` capabilities
+- Timezone is auto-detected from Tor exit node
+- Custom DNS/proxy settings are ignored when Tor is active
 
-## 🤝 Contributing
+### Lite
+A smaller image without the built-in Cloudflare bypasser. Ideal for:
 
-Contributions are welcome! Feel free to submit a Pull Request.
+- **External bypassers** - Already running FlareSolverr or ByParr for other services
+- **Fast downloads** - Using fast download sources
+- **Alternative sources only** - Exclusively using Prowlarr, AudiobookBay, IRC, or other sources
+- **Audiobooks** - Using Shelfmark exclusively for audiobooks
 
-## 📄 License
+```bash
+curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
+docker compose -f docker-compose.lite.yml up -d
+```
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
 
-## ⚠️ Important Disclaimers
+## 🔐 Authentication
+
+Authentication is optional but recommended for shared or exposed instances. Three authentication methods are available in Settings:
+
+**1. Single Username/Password**
+
+**2. Proxy (Forward) Authentication**
+
+Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
+
+**3. Calibre-Web Database**
+
+If you're running Calibre-Web, you can reuse its user database by mounting it:
+
+```yaml
+volumes:
+  - /path/to/calibre-web/app.db:/auth/app.db:ro
+```
+
+## Health Monitoring
+
+The application exposes a health endpoint at `/api/health` (no authentication required). Add a health check to your compose:
+
+```yaml
+healthcheck:
+  test: ["CMD", "curl", "-sf", "http://localhost:8084/api/health"]
+  interval: 30s
+  timeout: 30s
+  retries: 3
+```
+
+## Logging
+
+Logs are available via:
+- `docker logs <container-name>`
+- `/var/log/shelfmark/` inside the container (when `ENABLE_LOGGING=true`)
+
+Log level is configurable via Settings or `LOG_LEVEL` environment variable.
+
+## Development
+
+```bash
+# Frontend development
+make install     # Install dependencies
+make dev         # Start Vite dev server (localhost:5173)
+make build       # Production build
+make typecheck   # TypeScript checks
+
+# Backend (Docker)
+make up          # Start backend via docker-compose.dev.yml
+make down        # Stop services
+make refresh     # Rebuild and restart
+make restart     # Restart container
+```
+
+The frontend dev server proxies to the backend on port 8084.
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      Web Interface                          │
+│                 (React + TypeScript + Vite)                 │
+├─────────────────────────────────────────────────────────────┤
+│                      Flask Backend                          │
+│                   (REST API + WebSocket)                    │
+├───────────────────┬─────────────────────┬───────────────────┤
+│ Metadata Providers│   Download Queue    │  Cloudflare       │
+│                   │   & Orchestrator    │  Bypass           │
+├───────────────────┼─────────────────────┼───────────────────┤
+│ • Hardcover       │ • Task scheduling   │ • Internal        │
+│ • Open Library    │ • Progress tracking │ • External        │
+│                   │ • Retry logic       │   (FlareSolverr)  │
+├───────────────────┴─────────────────────┴───────────────────┤
+│                     Release Sources                         │
+├─────────────────────────────────────────────────────────────┤
+│ • Direct Download (Web Sources → Mirrors → Fallbacks)       │
+├─────────────────────────────────────────────────────────────┤
+│                     Network Layer                           │
+├─────────────────────────────────────────────────────────────┤
+│ • Auto DNS rotation  • Mirror failover  • Resume support    │
+└─────────────────────────────────────────────────────────────┘
+```
+
+The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered.
+
+## Contributing
+
+Contributions are welcome! Please file issues or submit pull requests on GitHub.
+
+> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR!
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## ⚠️ Disclaimers
 
 ### Copyright Notice
 
-While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for:
-
+This tool can access various sources including those that might contain copyrighted material. Users are responsible for:
 - Ensuring they have the right to download requested materials
 - Respecting copyright laws and intellectual property rights
 - Using the tool in compliance with their local regulations
 
-### Duplicate Downloads Warning
+### Library Integration
 
-Please note that the current version:
+Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads.
 
-- Does not check for existing files in the download directory
-- Does not verify if books already exist in your Calibre database
-- Exercise caution when requesting multiple books to avoid duplicates
-
-## 💬 Support
-
-For issues or questions, please file an issue on the GitHub repository.
+## Support
 
+For issues or questions, please [file an issue](https://github.com/calibrain/shelfmark/issues) on GitHub.

release-notes-v1.1.0.md

@@ -0,0 +1,47 @@
+## New Features
+
+### OIDC Authentication (#606, #612)
+- **OIDC login** with PKCE flow, auto-discovery, and group-based admin mapping
+- **Auto-provisioning** of OIDC users (configurable) and email-based account linking
+- **Password fallback** when OIDC is enabled to prevent admin lockout
+- Backwards compatible with all existing auth modes (no-auth, builtin, proxy, CWA)
+
+### Multi-User Support (#606, #612, #613)
+- **User management** -create, edit, and delete users with admin/user roles
+- **Per-user settings** -custom download destinations, BookLore library/path, email recipients, and `{User}` template variable
+- **Per-user download visibility** -non-admins only see their own downloads
+
+### Multi-User Request System (#615, #617, #620)
+- **Book request workflow** -users can request books with notes; admins review, approve, and fulfil requests
+- **Policy-based configuration** -set download/request/block policies per content type or per source (e.g. allow direct downloads, set Prowlarr to request-only)
+- **Per-user policy overrides** for tailored access control
+- **New Activity Sidebar** -replaces downloads sidebar, combining active downloads with requests; sidebar can now be pinned
+- Request retry support and admin-level request management
+
+### Notification Support (#618)
+- **Apprise-based notifications** for request events and download completions
+- Configurable globally or per user, with full customization of events and notification services
+- Expanded activity cards with detailed request info and file management
+
+### AudiobookBay Release Source (#619, #621, #623)
+- **New release source** -search AudiobookBay for audiobook torrents directly from the UI
+- Results include title, language, format, and size
+- Downloads via configured torrent client with audiobook-specific category support
+- Configurable hostname, max search pages, and rate limit delay
+
+### Email Output Mode (#603, #604)
+- **Email delivery** as an alternative output mode for downloaded books
+- Per-user email recipient configuration
+
+## Improvements
+- Admin-configurable visibility for self-settings options (delivery preferences, notifications) (#625)
+- BookLore Bookdrop API destination support as an alternative to specific library selection (#625)
+- Download path options for all torrent clients (#625)
+- Add tag support to qBittorrent downloads (#610 by @dawescc)
+- Add threading to file system operations for improved performance (#602)
+- Enhanced custom scripting -JSON download info, more consistent activation, decoupled from staging (#591)
+- Hardlink-before-move optimization for file transfers (#591)
+- New BookLore API file formats (#591)
+- Improved login cookie naming for reverse proxy compatibility (#591)
+- Fix Transmission URL parsing (#591)
+- Fix healthcheck starvation during large file processing (#591)

requirements-base.txt

@@ -12,3 +12,7 @@ gevent-websocket
 psutil
 emoji
 rarfile
+qbittorrent-api
+transmission-rpc
+authlib>=1.6.6,<1.7
+apprise>=1.9.0

requirements-shelfmark.txt

@@ -1,4 +1,4 @@
 pyvirtualdisplay
 pyautogui
-seleniumbase>=4.41.1
+seleniumbase==4.45.10
 python-xlib

scripts/fix_permissions.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fix permissions on all configured directories.
+
+This script is called by the entrypoint to ensure all user-configured
+directories have correct ownership. It reads directory paths from:
+- CONFIG_DIR environment variable
+- Config files in CONFIG_DIR/plugins/
+
+Outputs directory paths that need permission fixing (one per line).
+The entrypoint handles the actual chown operations.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def get_directories_from_config() -> set[str]:
+    """Extract all directory paths from config files."""
+    directories = set()
+
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    plugins_dir = config_dir / "plugins"
+
+    if not plugins_dir.exists():
+        return directories
+
+    # Keys that contain directory paths
+    directory_keys = {
+        # Main destinations
+        "DESTINATION",
+        "DESTINATION_AUDIOBOOK",
+        # Content type routing directories
+        "AA_CONTENT_TYPE_DIR_FICTION",
+        "AA_CONTENT_TYPE_DIR_NON_FICTION",
+        "AA_CONTENT_TYPE_DIR_UNKNOWN",
+        "AA_CONTENT_TYPE_DIR_MAGAZINE",
+        "AA_CONTENT_TYPE_DIR_COMIC",
+        "AA_CONTENT_TYPE_DIR_STANDARDS",
+        "AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
+        "AA_CONTENT_TYPE_DIR_OTHER",
+        # Legacy keys (in case of old configs)
+        "INGEST_DIR",
+        "INGEST_DIR_AUDIOBOOK",
+        "INGEST_DIR_BOOK_FICTION",
+        "INGEST_DIR_BOOK_NON_FICTION",
+        "INGEST_DIR_BOOK_UNKNOWN",
+        "INGEST_DIR_MAGAZINE",
+        "INGEST_DIR_COMIC_BOOK",
+        "INGEST_DIR_STANDARDS_DOCUMENT",
+        "INGEST_DIR_MUSICAL_SCORE",
+        "INGEST_DIR_OTHER",
+        "LIBRARY_PATH",
+        "LIBRARY_PATH_AUDIOBOOK",
+    }
+
+    # Read all JSON config files
+    for config_file in plugins_dir.glob("*.json"):
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+
+            for key in directory_keys:
+                if key in config:
+                    value = config[key]
+                    if value and isinstance(value, str) and value.startswith("/"):
+                        directories.add(value)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    return directories
+
+
+def main():
+    """Output all configured directories that exist."""
+    directories = get_directories_from_config()
+
+    # Filter to directories that actually exist
+    existing = []
+    for dir_path in directories:
+        path = Path(dir_path)
+        if path.exists() and path.is_dir():
+            existing.append(dir_path)
+
+    # Output one directory per line
+    for dir_path in sorted(existing):
+        print(dir_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_env_docs.py

@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""Generate markdown documentation for environment variables from the settings registry.
+
+This script extracts all settings that support environment variable configuration
+and generates a comprehensive markdown file documenting each option.
+
+Usage:
+    python scripts/generate_env_docs.py [--output path/to/output.md]
+
+The generated documentation includes:
+- Environment variable name
+- Description
+- Type (string, number, boolean, etc.)
+- Default value
+- Organizational grouping by settings tab/group
+"""
+
+import argparse
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# Add project root to path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+
+def get_field_type_name(field) -> str:
+    """Get a human-readable type name for a field."""
+    from shelfmark.core.settings_registry import (
+        CheckboxField,
+        MultiSelectField,
+        NumberField,
+        OrderableListField,
+        PasswordField,
+        SelectField,
+        TextField,
+    )
+
+    if isinstance(field, CheckboxField):
+        return "boolean"
+    elif isinstance(field, NumberField):
+        return "number"
+    elif isinstance(field, SelectField):
+        return "string (choice)"
+    elif isinstance(field, MultiSelectField):
+        return "string (comma-separated)"
+    elif isinstance(field, OrderableListField):
+        return "JSON array"
+    elif isinstance(field, PasswordField):
+        return "string (secret)"
+    elif isinstance(field, TextField):
+        return "string"
+    else:
+        return "string"
+
+
+def format_default_value(field) -> str:
+    """Format the default value for display."""
+    default = field.default
+
+    if default is None:
+        return "_none_"
+    elif isinstance(default, bool):
+        return f"`{str(default).lower()}`"
+    elif isinstance(default, (int, float)):
+        return f"`{default}`"
+    elif isinstance(default, str):
+        if default == "":
+            return "_empty string_"
+        return f"`{default}`"
+    elif isinstance(default, list):
+        if not default:
+            return "_empty list_"
+        # For simple lists, show comma-separated values
+        if all(isinstance(item, str) for item in default):
+            return f"`{','.join(default)}`"
+        # For complex lists (e.g., OrderableListField defaults), summarize
+        return f"_see UI for defaults_"
+    else:
+        return f"`{default}`"
+
+
+def get_select_options(field) -> Optional[List[str]]:
+    """Get the available options for a SelectField.
+
+    Returns options formatted as 'value (label)' or just 'value' if they match,
+    so users know the actual values to use in environment variables.
+    """
+    from shelfmark.core.settings_registry import SelectField
+
+    if not isinstance(field, SelectField):
+        return None
+
+    options = field.options
+    if callable(options):
+        try:
+            options = options()
+        except Exception:
+            return None
+
+    if not options:
+        return None
+
+    result = []
+    for opt in options:
+        value = opt.get("value", "")
+        label = opt.get("label", "")
+
+        # Format as "value (label)" unless they're the same or value is empty
+        if value == "":
+            result.append(f'`""` ({label})')
+        elif value == label or not label:
+            result.append(f"`{value}`")
+        else:
+            result.append(f"`{value}` ({label})")
+
+    return result
+
+
+def _generate_bootstrap_env_docs() -> List[str]:
+    """Generate documentation for bootstrap environment variables from env.py."""
+    # These are environment variables defined in env.py that are used before
+    # the settings registry is available
+    bootstrap_vars = [
+        {
+            "name": "CONFIG_DIR",
+            "description": "Directory for storing configuration files and plugin settings.",
+            "type": "string (path)",
+            "default": "/config",
+        },
+        {
+            "name": "LOG_ROOT",
+            "description": "Root directory for log files.",
+            "type": "string (path)",
+            "default": "/var/log/",
+        },
+        {
+            "name": "TMP_DIR",
+            "description": "Staging directory for downloads before moving to destination.",
+            "type": "string (path)",
+            "default": "/tmp/shelfmark",
+        },
+        {
+            "name": "ENABLE_LOGGING",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
+            "type": "boolean",
+            "default": "true",
+        },
+        {
+            "name": "FLASK_HOST",
+            "description": "Host address for the Flask web server.",
+            "type": "string",
+            "default": "0.0.0.0",
+        },
+        {
+            "name": "FLASK_PORT",
+            "description": "Port number for the Flask web server.",
+            "type": "number",
+            "default": "8084",
+        },
+        {
+            "name": "SESSION_COOKIE_SECURE",
+            "description": "Enable secure cookies (requires HTTPS).",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "CWA_DB_PATH",
+            "description": "Path to the Calibre-Web database for authentication integration.",
+            "type": "string (path)",
+            "default": "/auth/app.db",
+        },
+        {
+            "name": "DOCKERMODE",
+            "description": "Indicates the application is running inside a Docker container.",
+            "type": "boolean",
+            "default": "false",
+        },
+        {
+            "name": "ONBOARDING",
+            "description": "Show the onboarding wizard on first run. Set to false to skip (useful for ephemeral storage).",
+            "type": "boolean",
+            "default": "true",
+        },
+    ]
+
+    lines = [
+        "## Bootstrap Configuration",
+        "",
+        "These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
+        "",
+        "| Variable | Description | Type | Default |",
+        "|----------|-------------|------|---------|",
+    ]
+
+    for var in bootstrap_vars:
+        lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
+
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for var in bootstrap_vars:
+        lines.append(f"#### `{var['name']}`")
+        lines.append("")
+        lines.append(var["description"])
+        lines.append("")
+        lines.append(f"- **Type:** {var['type']}")
+        lines.append(f"- **Default:** `{var['default']}`")
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def generate_env_docs() -> str:
+    """Generate markdown documentation for all environment variables."""
+    # Import settings modules to ensure all settings are registered
+    import shelfmark.config.settings  # noqa: F401
+    import shelfmark.release_sources.irc.settings  # noqa: F401
+    import shelfmark.release_sources.prowlarr.settings  # noqa: F401
+    import shelfmark.metadata_providers.hardcover  # noqa: F401
+    import shelfmark.metadata_providers.openlibrary  # noqa: F401
+    import shelfmark.metadata_providers.googlebooks  # noqa: F401
+
+    from shelfmark.core.settings_registry import (
+        ActionButton,
+        HeadingField,
+        get_all_groups,
+        get_all_settings_tabs,
+    )
+
+    tabs = get_all_settings_tabs()
+    groups = {g.name: g for g in get_all_groups()}
+
+    # Organize tabs by group
+    grouped_tabs: Dict[Optional[str], List] = {None: []}
+    for group_name in groups:
+        grouped_tabs[group_name] = []
+
+    for tab in tabs:
+        group_name = tab.group
+        if group_name not in grouped_tabs:
+            grouped_tabs[group_name] = []
+        grouped_tabs[group_name].append(tab)
+
+    # Build markdown output
+    lines = [
+        "# Environment Variables",
+        "",
+        "This document lists all configuration options that can be set via environment variables.",
+        "",
+        "> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
+        "",
+        "## Table of Contents",
+        "",
+    ]
+
+    # Generate TOC
+    toc_entries = [
+        "- [Bootstrap Configuration](#bootstrap-configuration)",
+    ]
+
+    # Ungrouped tabs first
+    for tab in grouped_tabs.get(None, []):
+        anchor = tab.display_name.lower().replace(" ", "-")
+        toc_entries.append(f"- [{tab.display_name}](#{anchor})")
+
+    # Then grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if group_tabs:
+            anchor = group.display_name.lower().replace(" ", "-")
+            toc_entries.append(f"- [{group.display_name}](#{anchor})")
+            for tab in group_tabs:
+                sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
+                toc_entries.append(f"  - [{tab.display_name}](#{sub_anchor})")
+
+    lines.extend(toc_entries)
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Add bootstrap environment variables documentation
+    lines.extend(_generate_bootstrap_env_docs())
+
+    # Generate documentation for ungrouped tabs
+    for tab in grouped_tabs.get(None, []):
+        lines.extend(_generate_tab_docs(tab))
+
+    # Generate documentation for grouped tabs
+    for group_name, group in groups.items():
+        group_tabs = grouped_tabs.get(group_name, [])
+        if not group_tabs:
+            continue
+
+        lines.append(f"## {group.display_name}")
+        lines.append("")
+
+        for tab in group_tabs:
+            lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
+
+    return "\n".join(lines)
+
+
+def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
+    """Generate documentation for a single settings tab."""
+    from shelfmark.core.settings_registry import ActionButton, HeadingField
+
+    lines = []
+
+    # Section header
+    if group_prefix:
+        lines.append(f"### {group_prefix}: {tab.display_name}")
+        anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
+    else:
+        lines.append(f"## {tab.display_name}")
+
+    lines.append("")
+
+    # Collect env-supported fields
+    env_fields = []
+    for field in tab.fields:
+        # Skip non-value fields
+        if isinstance(field, (ActionButton, HeadingField)):
+            continue
+
+        # Skip fields that don't support ENV vars
+        if not getattr(field, "env_supported", True):
+            continue
+
+        env_fields.append(field)
+
+    if not env_fields:
+        lines.append("_No environment variables for this section._")
+        lines.append("")
+        return lines
+
+    # Generate table
+    lines.append("| Variable | Description | Type | Default |")
+    lines.append("|----------|-------------|------|---------|")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        description = field.description or field.label
+        # Clean up description for table (remove newlines, escape pipes)
+        description = description.replace("\n", " ").replace("|", "\\|").strip()
+
+        field_type = get_field_type_name(field)
+        default = format_default_value(field)
+
+        lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
+
+    lines.append("")
+
+    # Add detailed documentation for each field
+    lines.append("<details>")
+    lines.append("<summary>Detailed descriptions</summary>")
+    lines.append("")
+
+    for field in env_fields:
+        env_var = field.get_env_var_name()
+        lines.append(f"#### `{env_var}`")
+        lines.append("")
+        lines.append(f"**{field.label}**")
+        lines.append("")
+
+        if field.description:
+            lines.append(field.description)
+            lines.append("")
+
+        lines.append(f"- **Type:** {get_field_type_name(field)}")
+        lines.append(f"- **Default:** {format_default_value(field)}")
+
+        if getattr(field, "required", False):
+            lines.append("- **Required:** Yes")
+
+        if getattr(field, "requires_restart", False):
+            lines.append("- **Requires restart:** Yes")
+
+        # Show options for SelectField
+        options = get_select_options(field)
+        if options:
+            lines.append(f"- **Options:** {', '.join(options)}")
+
+        # Show constraints for NumberField
+        from shelfmark.core.settings_registry import NumberField
+        if isinstance(field, NumberField):
+            constraints = []
+            if field.min_value is not None:
+                constraints.append(f"min: {field.min_value}")
+            if field.max_value is not None:
+                constraints.append(f"max: {field.max_value}")
+            if constraints:
+                lines.append(f"- **Constraints:** {', '.join(constraints)}")
+
+        lines.append("")
+
+    lines.append("</details>")
+    lines.append("")
+
+    return lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate markdown documentation for environment variables"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=project_root / "docs" / "environment-variables.md",
+        help="Output file path (default: docs/environment-variables.md)",
+    )
+    parser.add_argument(
+        "--stdout",
+        action="store_true",
+        help="Print to stdout instead of file",
+    )
+    args = parser.parse_args()
+
+    docs = generate_env_docs()
+
+    if args.stdout:
+        print(docs)
+    else:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(docs)
+        print(f"Generated: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_clients.py

@@ -0,0 +1,555 @@
+#!/usr/bin/env python3
+"""
+Test script for download client implementations.
+
+Usage:
+    1. Start the test stack:
+       docker compose -f docker-compose.test-clients.yml up -d
+
+    2. Wait for containers to initialize (first run takes ~30s)
+
+    3. Run this script to verify clients are accessible:
+       python scripts/test_clients.py
+
+    4. Access cwabd at http://localhost:8084
+       - Go to Settings > Prowlarr > Download Clients
+       - Select a client from the dropdown
+       - Click "Test Connection" to verify
+
+Web UIs:
+    - cwabd:        http://localhost:8084
+    - qBittorrent:  http://localhost:8080
+    - Transmission: http://localhost:9091
+    - Deluge:       http://localhost:8112
+    - NZBGet:       http://localhost:6789
+    - SABnzbd:      http://localhost:8085
+    - rTorrent:     http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
+
+Prerequisites (for running this script locally):
+    pip install requests transmission-rpc qbittorrent-api
+
+First-Time Setup:
+    qBittorrent:
+        - Check container logs for temporary password: docker logs test-qbittorrent
+        - Login at http://localhost:8080, change password to something known
+        - Default username is 'admin'
+
+    Transmission:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    Deluge:
+        - Access Web UI at http://localhost:8112 (default password: deluge)
+
+    NZBGet:
+        - No setup needed, credentials pre-configured (admin/admin)
+
+    SABnzbd:
+        - Complete the setup wizard at http://localhost:8085
+        - API key will be auto-detected by this script
+        - In cwabd, copy API key from SABnzbd Config > General
+"""
+
+import sys
+import time
+from xmlrpc import client
+
+# Test configuration - matches docker-compose.test-clients.yml
+CONFIG = {
+    # Usenet clients
+    "nzbget": {
+        "url": "http://localhost:6789",
+        "username": "admin",
+        "password": "admin",
+    },
+    "sabnzbd": {
+        "url": "http://localhost:8085",
+        "api_key": None,  # Will be read from config on first run
+    },
+    # Torrent clients
+    "qbittorrent": {
+        "url": "http://localhost:8080",
+        "username": "admin",
+        "password": "5NCngsHXm",  # Temp password from: docker logs test-qbittorrent | grep password
+    },
+    "transmission": {
+        "url": "http://localhost:9091",
+        "username": "admin",
+        "password": "admin",
+    },
+    "deluge": {
+        "url": "http://localhost:8112",
+        "password": "deluge",
+    },
+    "rtorrent": {
+        "url": "http://localhost:8000/RPC2",
+    },
+}
+
+# Test magnet link (Ubuntu ISO - legal, small metadata)
+TEST_MAGNET = "magnet:?xt=urn:btih:3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0&dn=ubuntu-22.04.3-live-server-amd64.iso"
+
+
+def test_nzbget():
+    """Test NZBGet connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing NZBGet")
+    print("=" * 50)
+
+    url = CONFIG["nzbget"]["url"]
+    username = CONFIG["nzbget"]["username"]
+    password = CONFIG["nzbget"]["password"]
+
+    try:
+        # Test connection via JSON-RPC
+        rpc_url = f"{url}/jsonrpc"
+        response = requests.post(
+            rpc_url,
+            json={"method": "version", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("result", "unknown")
+        print(f"  Connected to NZBGet {version}")
+
+        # Test status
+        response = requests.post(
+            rpc_url,
+            json={"method": "status", "params": []},
+            auth=(username, password),
+            timeout=10,
+        )
+        status = response.json().get("result", {})
+        print(f"  Server state: {'Paused' if status.get('ServerPaused') else 'Running'}")
+        print(f"  Downloads in queue: {status.get('DownloadedSizeMB', 0)} MB downloaded")
+
+        print("  SUCCESS: NZBGet is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to NZBGet")
+        print("  Is the container running? docker ps | grep nzbget")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_sabnzbd():
+    """Test SABnzbd connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing SABnzbd")
+    print("=" * 50)
+
+    url = CONFIG["sabnzbd"]["url"]
+    api_key = CONFIG["sabnzbd"]["api_key"]
+
+    # Try to get API key from config if not set
+    if not api_key:
+        try:
+            import os
+            ini_path = ".local/test-clients/sabnzbd/config/sabnzbd.ini"
+            if os.path.exists(ini_path):
+                with open(ini_path) as f:
+                    for line in f:
+                        if line.startswith("api_key"):
+                            api_key = line.split("=")[1].strip()
+                            print(f"  Found API key in config: {api_key[:8]}...")
+                            break
+        except Exception as e:
+            print(f"  Could not read API key from config: {e}")
+
+    if not api_key:
+        print("  ERROR: No API key configured")
+        print("  Please access http://localhost:8085 and complete initial setup")
+        print("  Then copy the API key from Config > General")
+        return False
+
+    try:
+        # Test connection
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "version", "output": "json"},
+            timeout=10,
+        )
+        response.raise_for_status()
+        result = response.json()
+        version = result.get("version", "unknown")
+        print(f"  Connected to SABnzbd {version}")
+
+        # Test queue status
+        response = requests.get(
+            f"{url}/api",
+            params={"apikey": api_key, "mode": "queue", "output": "json"},
+            timeout=10,
+        )
+        queue = response.json().get("queue", {})
+        print(f"  Queue status: {queue.get('status', 'unknown')}")
+        print(f"  Items in queue: {len(queue.get('slots', []))}")
+
+        print("  SUCCESS: SABnzbd is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to SABnzbd")
+        print("  Is the container running? docker ps | grep sabnzbd")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_qbittorrent():
+    """Test qBittorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing qBittorrent")
+    print("=" * 50)
+
+    try:
+        import qbittorrentapi
+
+        url = CONFIG["qbittorrent"]["url"]
+        username = CONFIG["qbittorrent"]["username"]
+        password = CONFIG["qbittorrent"]["password"]
+
+        # Parse URL for host/port
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+
+        client = qbittorrentapi.Client(
+            host=parsed.hostname,
+            port=parsed.port or 8080,
+            username=username,
+            password=password,
+        )
+
+        # Test connection
+        client.auth_log_in()
+        version = client.app.version
+        print(f"  Connected to qBittorrent {version}")
+
+        # Get torrent list
+        torrents = client.torrents_info()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        result = client.torrents_add(urls=TEST_MAGNET, is_paused=True)
+        if result == "Ok.":
+            # Wait a moment for it to be added
+            time.sleep(1)
+            torrents = client.torrents_info()
+            if torrents:
+                test_torrent = torrents[-1]  # Most recently added
+                print(f"  Added test torrent: {test_torrent.name[:50]}...")
+                print(f"  Status: {test_torrent.state}")
+
+                # Remove it
+                client.torrents_delete(torrent_hashes=test_torrent.hash, delete_files=True)
+                print("  Removed test torrent")
+        else:
+            print(f"  Add result: {result}")
+
+        print("  SUCCESS: qBittorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: qbittorrent-api not installed")
+        print("  Run: pip install qbittorrent-api")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Forbidden" in str(e) or "401" in str(e):
+            print("\n  Authentication failed. Check password:")
+            print("  1. docker logs test-qbittorrent | grep password")
+            print("  2. Login to http://localhost:8080 and set a known password")
+        return False
+
+
+def test_transmission():
+    """Test Transmission connection."""
+    print("\n" + "=" * 50)
+    print("Testing Transmission")
+    print("=" * 50)
+
+    try:
+        from transmission_rpc import Client
+        from urllib.parse import urlparse
+
+        url = CONFIG["transmission"]["url"]
+        parsed = urlparse(url)
+
+        client = Client(
+            host=parsed.hostname,
+            port=parsed.port or 9091,
+            username=CONFIG["transmission"]["username"],
+            password=CONFIG["transmission"]["password"],
+        )
+
+        # Test connection
+        session = client.get_session()
+        print(f"  Connected to Transmission {session.version}")
+
+        # Get torrent list
+        torrents = client.get_torrents()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent = client.add_torrent(TEST_MAGNET, paused=True)
+        print(f"  Added test torrent: {torrent.name[:50]}...")
+
+        # Get status
+        status = client.get_torrent(torrent.id)
+        print(f"  Status: {status.status} ({status.percent_done * 100:.1f}%)")
+
+        # Remove it
+        client.remove_torrent(torrent.id, delete_data=True)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: Transmission is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: transmission-rpc not installed")
+        print("  Run: pip install transmission-rpc")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        return False
+
+
+def test_deluge():
+    """Test Deluge Web UI (JSON-RPC) connection."""
+    import requests
+
+    print("\n" + "=" * 50)
+    print("Testing Deluge")
+    print("=" * 50)
+
+    base_url = CONFIG["deluge"]["url"].rstrip("/")
+    password = CONFIG["deluge"]["password"]
+    rpc_url = f"{base_url}/json"
+
+    def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
+        payload = {"id": rpc_id, "method": method, "params": list(params)}
+        resp = session.post(rpc_url, json=payload, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if data.get("error"):
+            err = data["error"]
+            if isinstance(err, dict):
+                raise Exception(err.get("message") or str(err))
+            raise Exception(str(err))
+        return data.get("result")
+
+    try:
+        session = requests.Session()
+
+        # Authenticate to Deluge Web
+        if rpc_call(session, 1, "auth.login", password) is not True:
+            raise Exception("Authentication failed (check Deluge Web UI password)")
+
+        # Ensure Deluge Web is connected to a daemon
+        if rpc_call(session, 2, "web.connected") is not True:
+            hosts = rpc_call(session, 3, "web.get_hosts") or []
+            if not hosts:
+                raise Exception(
+                    "Deluge Web UI isn't connected to Deluge core (no hosts configured). "
+                    "Add/connect a daemon in Deluge Web UI → Connection Manager."
+                )
+
+            host_id = hosts[0][0]
+            for entry in hosts:
+                if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
+                    host_id = entry[0]
+                    break
+
+            rpc_call(session, 4, "web.connect", host_id)
+
+            if rpc_call(session, 5, "web.connected") is not True:
+                raise Exception(
+                    "Deluge Web UI couldn't connect to Deluge core. "
+                    "Check Deluge Web UI → Connection Manager."
+                )
+
+        version = rpc_call(session, 6, "daemon.info")
+        print(f"  Connected to Deluge {version}")
+
+        torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+        torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
+
+        if torrent_id:
+            torrent_id = str(torrent_id)
+            print(f"  Added test torrent: {torrent_id[:20]}...")
+
+            status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
+            state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
+            progress = status.get("progress", 0) if isinstance(status, dict) else 0
+            print(f"  Status: {state} ({progress:.1f}%)")
+
+            rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
+            print("  Removed test torrent")
+        else:
+            print("  WARNING: Could not add test torrent")
+
+        print("  SUCCESS: Deluge is working!")
+        return True
+
+    except requests.exceptions.ConnectionError:
+        print("  ERROR: Could not connect to Deluge Web UI")
+        print("  Is the container running? docker ps | grep deluge")
+        return False
+    except requests.exceptions.Timeout:
+        print("  ERROR: Deluge Web UI connection timed out")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "auth" in str(e).lower() or "login" in str(e).lower():
+            print("  Check Deluge Web UI password (default: deluge)")
+        return False
+
+def test_rtorrent():
+    """Test rTorrent connection."""
+    print("\n" + "=" * 50)
+    print("Testing rTorrent")
+    print("=" * 50)
+
+    try:
+        import xmlrpc.client
+
+        url = "http://localhost:8000/RPC2"
+        client = xmlrpc.client.ServerProxy(url)
+
+        # Test connection
+        version = client.system.library_version()
+        print(f"  Connected to rTorrent {version}")
+
+        # default download directory test
+        default_dir = client.directory.default()
+        print(f"  Default download directory: {default_dir}")
+
+        # Get torrent list
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")
+
+        # Test adding a torrent (then remove it)
+        print("  Testing add/remove torrent...")
+
+        label = "automated"
+
+        commands = []
+        if label:
+            commands.append(f"d.custom1.set={label}")
+
+        download_dir = "/downloads"
+        if download_dir:
+            commands.append(f"d.directory_base.set={download_dir}")
+
+        # rtorrent is weird in that it doesn't return the torrent ID/hash on add
+        client.load.start("", TEST_MAGNET, ";".join(commands))
+        
+        # but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
+        torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
+        print(f"  Added test torrent: {torrent_id}")
+
+        torrents = client.download_list()
+        print(f"  Active torrents: {len(torrents)}")        
+
+        torrent_list = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal={{d.hash=,cat={torrent_id}}}"
+            "d.hash=",
+            "d.state=",
+            "d.completed_bytes=",
+            "d.size_bytes=",
+            "d.down.rate=",
+            "d.up.rate=",
+            "d.custom1=",
+            "d.complete=",
+        )
+        torrent = torrent_list[0]
+
+        if not torrent:
+            print("  ERROR: Could not find added torrent in list")
+            return False
+        
+        # let's test the base path call
+        details = client.d.multicall.filtered(
+            "",
+            "default",
+            f"equal=d.hash=,cat={torrent_id}",
+            "d.base_path=",
+        )
+
+        base_path = details[0][0] if details else None
+
+        print(f"  Base path: {base_path}")
+        client.d.erase(torrent_id)
+        print("  Removed test torrent")
+
+        print("  SUCCESS: rTorrent is working!")
+        return True
+
+    except ImportError:
+        print("  ERROR: xmlrpc.client not available")
+        return False
+    except Exception as e:
+        print(f"  ERROR: {e}")
+        if "Connection refused" in str(e):
+            print("  Is the container running? docker ps | grep rtorrent")
+        return False
+
+
+def main():
+    print("Download Client Test Suite")
+    print("=" * 50)
+    print("Make sure containers are running:")
+    print("  docker compose -f docker-compose.test-clients.yml up -d")
+
+    results = {}
+
+    # Test usenet clients
+    print("\n" + "=" * 50)
+    print("USENET CLIENTS")
+    print("=" * 50)
+    results["nzbget"] = test_nzbget()
+    results["sabnzbd"] = test_sabnzbd()
+
+    # Test torrent clients
+    print("\n" + "=" * 50)
+    print("TORRENT CLIENTS")
+    print("=" * 50)
+    results["qbittorrent"] = test_qbittorrent()
+    results["transmission"] = test_transmission()
+    results["deluge"] = test_deluge()
+    results["rtorrent"] = test_rtorrent()
+
+    # Summary
+    print("\n" + "=" * 50)
+    print("SUMMARY")
+    print("=" * 50)
+
+    for client, success in results.items():
+        status = "PASS" if success else "FAIL"
+        print(f"  {client}: {status}")
+
+    passed = sum(results.values())
+    total = len(results)
+    print(f"\n  Total: {passed}/{total} passed")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

shelfmark/__init__.py

@@ -0,0 +1 @@
+"""Shelfmark - book search and download service."""

shelfmark/__main__.py

@@ -0,0 +1,8 @@
+"""Package entry point for `python -m shelfmark`."""
+
+from shelfmark.main import app, socketio
+from shelfmark.config.env import FLASK_HOST, FLASK_PORT
+from shelfmark.core.config import config
+
+if __name__ == "__main__":
+    socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))

shelfmark/api/websocket.py

@@ -4,7 +4,7 @@ import logging
 import threading
 from typing import Optional, Dict, Any, Callable, List
 
-from flask_socketio import SocketIO
+from flask_socketio import SocketIO, join_room, leave_room
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +20,10 @@ class WebSocketManager:
         self._on_first_connect_callbacks: List[Callable[[], None]] = []
         self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
         self._needs_rewarm = False  # Flag to trigger warmup callbacks on next connect
+        self._user_rooms: Dict[str, int] = {}  # room_name -> ref count
+        self._sid_rooms: Dict[str, str] = {}  # sid -> room_name
+        self._rooms_lock = threading.Lock()
+        self._queue_status_fn: Optional[Callable] = None  # Reference to queue_status()
 
     def init_app(self, app, socketio: SocketIO):
         """Initialize the WebSocket manager with Flask-SocketIO instance."""
@@ -28,29 +32,17 @@ class WebSocketManager:
         logger.info("WebSocket manager initialized")
 
     def register_on_first_connect(self, callback: Callable[[], None]):
-        """Register a callback to be called when the first client connects.
-
-        This is useful for warming up resources (like the Cloudflare bypasser)
-        when a user starts using the web UI.
-        """
+        """Register a callback for when the first client connects."""
         self._on_first_connect_callbacks.append(callback)
         logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
 
     def register_on_all_disconnect(self, callback: Callable[[], None]):
-        """Register a callback to be called when all clients disconnect.
-
-        This can be used to trigger cleanup or resource release.
-        """
+        """Register a callback for when all clients disconnect."""
         self._on_all_disconnect_callbacks.append(callback)
         logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
 
     def request_warmup_on_next_connect(self):
-        """Request that warmup callbacks be triggered on the next client connect.
-
-        This is used when resources (like the Cloudflare bypasser) shut down due to
-        inactivity while clients are still connected. The next connect event should
-        trigger warmup even though it's not technically the "first" connection.
-        """
+        """Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
         with self._connection_lock:
             self._needs_rewarm = True
             logger.debug("Warmup requested for next client connect")
@@ -112,19 +104,86 @@ class WebSocketManager:
         """Check if WebSocket is enabled and ready."""
         return self._enabled and self.socketio is not None
 
+    def set_queue_status_fn(self, fn: Callable):
+        """Set the queue_status function reference for per-room filtering."""
+        self._queue_status_fn = fn
+
+    def _increment_user_room_locked(self, room: str):
+        self._user_rooms[room] = self._user_rooms.get(room, 0) + 1
+
+    def _decrement_user_room_locked(self, room: str):
+        count = self._user_rooms.get(room, 1) - 1
+        if count <= 0:
+            self._user_rooms.pop(room, None)
+        else:
+            self._user_rooms[room] = count
+
+    def _set_sid_room_locked(self, sid: str, room: Optional[str]):
+        current_room = self._sid_rooms.get(sid)
+        if current_room == room:
+            return
+
+        if current_room is not None:
+            leave_room(current_room, sid=sid)
+            if current_room.startswith("user_"):
+                self._decrement_user_room_locked(current_room)
+            self._sid_rooms.pop(sid, None)
+
+        if room is not None:
+            join_room(room, sid=sid)
+            self._sid_rooms[sid] = room
+            if room.startswith("user_"):
+                self._increment_user_room_locked(room)
+
+    def sync_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
+        """Ensure a SID is in exactly one room matching the current session scope."""
+        room: Optional[str] = None
+        if is_admin:
+            room = "admins"
+        elif db_user_id is not None:
+            room = f"user_{db_user_id}"
+
+        with self._rooms_lock:
+            self._set_sid_room_locked(sid, room)
+
+    def join_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
+        """Join the appropriate room based on user role."""
+        self.sync_user_room(sid, is_admin, db_user_id)
+
+    def leave_user_room(self, sid: str, is_admin: bool = False, db_user_id: Optional[int] = None):
+        """Leave whichever room the SID currently belongs to."""
+        del is_admin, db_user_id  # Backward-compatible signature; routing is SID-based.
+        with self._rooms_lock:
+            self._set_sid_room_locked(sid, None)
+
     def broadcast_status_update(self, status_data: Dict[str, Any]):
-        """Broadcast status update to all connected clients."""
+        """Broadcast status update to all connected clients, filtered by user room."""
         if not self.is_enabled():
             return
 
         try:
-            # When calling socketio.emit() outside event handlers, it broadcasts by default
-            self.socketio.emit('status_update', status_data)
-            logger.debug(f"Broadcasted status update to all clients")
+            # Admins (and no-auth users) get full status
+            self.socketio.emit('status_update', status_data, to="admins")
+
+            # Each user room gets filtered status
+            with self._rooms_lock:
+                active_rooms = list(self._user_rooms.keys())
+
+            if active_rooms and self._queue_status_fn:
+                for room in active_rooms:
+                    try:
+                        # Extract user_id from room name "user_123"
+                        uid = int(room.split("_", 1)[1])
+                        filtered = self._queue_status_fn(user_id=uid)
+                        self.socketio.emit('status_update', filtered, to=room)
+                    except Exception as e:
+                        logger.error(f"Failed to send status update for room {room}: {e}")
+
+            logger.debug("Broadcasted status update to all rooms")
         except Exception as e:
             logger.error(f"Error broadcasting status update: {e}")
 
-    def broadcast_download_progress(self, book_id: str, progress: float, status: str):
+    def broadcast_download_progress(self, book_id: str, progress: float, status: str, user_id: Optional[int] = None):
         """Broadcast download progress update for a specific book."""
         if not self.is_enabled():
             return
@@ -135,8 +194,14 @@ class WebSocketManager:
                 'progress': progress,
                 'status': status
             }
-            # When calling socketio.emit() outside event handlers, it broadcasts by default
-            self.socketio.emit('download_progress', data)
+            # Admins always see all progress
+            self.socketio.emit('download_progress', data, to="admins")
+            # If task belongs to a specific user, send to their room too
+            if user_id is not None:
+                room = f"user_{user_id}"
+                with self._rooms_lock:
+                    if room in self._user_rooms:
+                        self.socketio.emit('download_progress', data, to=room)
             logger.debug(f"Broadcasted progress for book {book_id}: {progress}%")
         except Exception as e:
             logger.error(f"Error broadcasting download progress: {e}")
@@ -157,6 +222,30 @@ class WebSocketManager:
         except Exception as e:
             logger.error(f"Error broadcasting notification: {e}")
 
+    def broadcast_search_status(
+        self,
+        source: str,
+        provider: str,
+        book_id: str,
+        message: str,
+        phase: str = 'searching'
+    ):
+        """Broadcast search status update for a release source search."""
+        if not self.is_enabled():
+            return
+
+        try:
+            data = {
+                'source': source,
+                'provider': provider,
+                'book_id': book_id,
+                'message': message,
+                'phase': phase,
+            }
+            self.socketio.emit('search_status', data)
+        except Exception as e:
+            logger.error(f"Error broadcasting search status: {e}")
+
 
 # Global WebSocket manager instance
 ws_manager = WebSocketManager()

shelfmark/bypass/__init__.py

@@ -0,0 +1,5 @@
+"""Cloudflare bypass utilities."""
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""

shelfmark/bypass/external_bypasser.py

@@ -0,0 +1,128 @@
+"""External Cloudflare bypasser using FlareSolverr."""
+
+import random
+import time
+from threading import Event
+from typing import TYPE_CHECKING, Optional
+
+import requests
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.utils import normalize_http_url
+
+if TYPE_CHECKING:
+    from shelfmark.download import network
+
+logger = setup_logger(__name__)
+
+# Timeout constants (seconds)
+CONNECT_TIMEOUT = 10
+MAX_READ_TIMEOUT = 120
+READ_TIMEOUT_BUFFER = 15
+
+# Retry settings
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
+
+
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service. Returns HTML or None."""
+    raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
+    bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
+    bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
+
+    bypasser_url = normalize_http_url(raw_bypasser_url)
+    if not bypasser_url or not bypasser_path:
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
+        return None
+
+    read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+
+    try:
+        response = requests.post(
+            f"{bypasser_url}{bypasser_path}",
+            headers={"Content-Type": "application/json"},
+            json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
+            timeout=(CONNECT_TIMEOUT, read_timeout)
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+
+        solution = result.get('solution')
+        html = solution.get('response', '') if solution else ''
+
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+
+        return html
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+
+    return None
+
+
+def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
+    """Check if operation was cancelled and raise exception if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(f"External bypasser cancelled {context}")
+        raise BypassCancelledException("Bypass cancelled")
+
+
+def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
+    """Sleep for the specified duration, checking for cancellation each second."""
+    for _ in range(int(seconds)):
+        _check_cancelled(cancel_flag, "during backoff")
+        time.sleep(1)
+    remaining = seconds - int(seconds)
+    if remaining > 0:
+        time.sleep(remaining)
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional["network.AAMirrorSelector"] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML via external bypasser with retries and mirror rotation."""
+    from shelfmark.download import network as network_module
+
+    sel = selector or network_module.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        _check_cancelled(cancel_flag, "by user")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        _sleep_with_cancellation(delay, cancel_flag)
+
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

shelfmark/bypass/fingerprint.py

@@ -0,0 +1,57 @@
+"""Browser fingerprint profile management for bypass stealth."""
+
+import random
+from typing import Optional
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+COMMON_RESOLUTIONS = [
+    (1920, 1080, 0.35),  
+    (1366, 768, 0.18),   
+    (1536, 864, 0.10),   
+    (1440, 900, 0.08),   
+    (1280, 720, 0.07),   
+    (1600, 900, 0.06),   
+    (1280, 800, 0.05),   
+    (2560, 1440, 0.04),  
+    (1680, 1050, 0.04),  
+    (1920, 1200, 0.03),  
+]
+
+# Current screen size (module-level singleton)
+_current_screen_size: Optional[tuple[int, int]] = None
+
+
+def get_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    if _current_screen_size is None:
+        _current_screen_size = _generate_screen_size()
+        logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
+    return _current_screen_size
+
+
+def rotate_screen_size() -> tuple[int, int]:
+    global _current_screen_size
+    old_size = _current_screen_size
+    _current_screen_size = _generate_screen_size()
+    width, height = _current_screen_size
+
+    if old_size:
+        logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
+    else:
+        logger.info(f"Generated screen size: {width}x{height}")
+
+    return _current_screen_size
+
+
+def clear_screen_size() -> None:
+    global _current_screen_size
+    _current_screen_size = None
+
+
+def _generate_screen_size() -> tuple[int, int]:
+    resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
+    weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
+    return random.choices(resolutions, weights=weights)[0]

shelfmark/bypass/internal_bypasser.py

@@ -0,0 +1,974 @@
+import asyncio
+import os
+import random
+import signal
+import socket
+import subprocess
+import threading
+import time
+import traceback
+from datetime import datetime
+from threading import Event
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+import requests
+from seleniumbase import cdp_driver
+
+from shelfmark.bypass import BypassCancelledException
+from shelfmark.bypass.fingerprint import get_screen_size
+from shelfmark.config import env
+from shelfmark.config.env import LOG_DIR
+from shelfmark.config.settings import RECORDING_DIR
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download import network
+from shelfmark.download.network import get_proxies
+
+logger = setup_logger(__name__)
+
+# Challenge detection indicators
+CLOUDFLARE_INDICATORS = [
+    "just a moment",
+    "verify you are human",
+    "verifying you are human",
+    "cloudflare.com/products/turnstile",
+]
+
+DDOS_GUARD_INDICATORS = [
+    "ddos-guard",
+    "ddos guard",
+    "checking your browser before accessing",
+    "complete the manual check to continue",
+    "could not verify your browser automatically",
+]
+
+DISPLAY = {
+    "ffmpeg": None,
+    "ffmpeg_output": None,
+}
+LOCKED = threading.Lock()
+
+
+class _CdpWorker:
+    def __init__(self) -> None:
+        self._thread: Optional[threading.Thread] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._ready = threading.Event()
+        self._lock = threading.Lock()
+
+    def _run(self) -> None:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        self._loop = loop
+        self._ready.set()
+        loop.run_forever()
+        try:
+            pending = asyncio.all_tasks(loop)
+            for task in pending:
+                task.cancel()
+            if pending:
+                loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+        except Exception:
+            pass
+        finally:
+            loop.close()
+
+    def start(self) -> None:
+        with self._lock:
+            if self._thread and self._thread.is_alive():
+                return
+            self._ready.clear()
+            self._thread = threading.Thread(
+                target=self._run,
+                name="cdp-worker",
+                daemon=True,
+            )
+            self._thread.start()
+        if not self._ready.wait(timeout=10):
+            raise RuntimeError("CDP worker loop failed to start")
+
+    def run(self, coro: Any, timeout: Optional[float] = None) -> Any:
+        self.start()
+        if not self._loop or self._loop.is_closed():
+            raise RuntimeError("CDP worker loop not available")
+        future = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        return future.result(timeout=timeout)
+
+
+_CDP_WORKER = _CdpWorker()
+
+# Cookie storage - shared with requests library for Cloudflare bypass
+# Structure: {domain: {cookie_name: {value, expiry, ...}}}
+_cf_cookies: dict[str, dict] = {}
+_cf_cookies_lock = threading.Lock()
+
+# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
+_cf_user_agents: dict[str, str] = {}
+
+# Protection cookie names we care about (Cloudflare and DDoS-Guard)
+CF_COOKIE_NAMES = {'cf_clearance', '__cf_bm', 'cf_chl_2', 'cf_chl_prog'}
+DDG_COOKIE_NAMES = {'__ddg1_', '__ddg2_', '__ddg5_', '__ddg8_', '__ddg9_', '__ddg10_', '__ddgid_', '__ddgmark_', 'ddg_last_challenge'}
+
+# Domains requiring full session cookies (not just protection cookies)
+FULL_COOKIE_DOMAINS = {'z-lib.fm', 'z-lib.gs', 'z-lib.id', 'z-library.sk', 'zlibrary-global.se'}
+
+
+def _get_base_domain(domain: str) -> str:
+    """Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
+    return '.'.join(domain.split('.')[-2:]) if '.' in domain else domain
+
+
+def _should_extract_cookie(name: str, extract_all: bool) -> bool:
+    """Determine if a cookie should be extracted based on its name."""
+    if extract_all:
+        return True
+    is_cf = name in CF_COOKIE_NAMES or name.startswith('cf_')
+    is_ddg = name in DDG_COOKIE_NAMES or name.startswith('__ddg')
+    return is_cf or is_ddg
+
+
+def _store_extracted_cookies(
+    *,
+    url: str,
+    cookies: list[Any],
+    user_agent: Optional[str] = None,
+) -> None:
+    """Store filtered bypass cookies (and optional UA) for a URL domain."""
+    parsed = urlparse(url)
+    domain = parsed.hostname or ""
+    if not domain:
+        return
+
+    base_domain = _get_base_domain(domain)
+    extract_all = base_domain in FULL_COOKIE_DOMAINS
+
+    cookies_found: dict[str, dict[str, Any]] = {}
+    for cookie in cookies:
+        name = getattr(cookie, "name", "") or ""
+        if not _should_extract_cookie(name, extract_all):
+            continue
+        expires = getattr(cookie, "expires", None)
+        if expires is not None and expires <= 0:
+            expires = None
+        cookies_found[name] = {
+            "value": getattr(cookie, "value", ""),
+            "domain": getattr(cookie, "domain", None) or domain,
+            "path": getattr(cookie, "path", None) or "/",
+            "expiry": expires,
+            "secure": bool(getattr(cookie, "secure", True)),
+            "httpOnly": True,
+        }
+
+    if not cookies_found:
+        return
+
+    with _cf_cookies_lock:
+        _cf_cookies[base_domain] = cookies_found
+        if user_agent:
+            _cf_user_agents[base_domain] = user_agent
+            logger.debug(f"Stored UA for {base_domain}: {str(user_agent)[:60]}...")
+        else:
+            logger.debug(f"No UA captured for {base_domain}")
+
+    cookie_type = "all" if extract_all else "protection"
+    logger.debug(f"Extracted {len(cookies_found)} {cookie_type} cookies for {base_domain}")
+
+
+async def _extract_cookies_from_cdp(driver, page, url: str) -> None:
+    """Extract cookies from a CDP browser after successful bypass."""
+    try:
+        try:
+            all_cookies = await driver.cookies.get_all(requests_cookie_format=True)
+        except Exception as e:
+            logger.debug(f"Failed to get cookies via CDP: {e}")
+            return
+
+        try:
+            user_agent = await page.evaluate("navigator.userAgent")
+        except Exception:
+            user_agent = None
+
+        _store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
+
+    except Exception as e:
+        logger.debug(f"Failed to extract cookies: {e}")
+
+def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
+    """Get stored cookies for a domain. Returns empty dict if none available."""
+    if not domain:
+        return {}
+
+    base_domain = _get_base_domain(domain)
+
+    with _cf_cookies_lock:
+        cookies = _cf_cookies.get(base_domain, {})
+        if not cookies:
+            return {}
+
+        cf_clearance = cookies.get('cf_clearance', {})
+        if cf_clearance:
+            expiry = cf_clearance.get('expiry')
+            if expiry is None:
+                expiry = cf_clearance.get('expires')
+            if expiry and expiry > 0 and time.time() > expiry:
+                logger.debug(f"CF cookies expired for {base_domain}")
+                _cf_cookies.pop(base_domain, None)
+                return {}
+
+        return {name: c['value'] for name, c in cookies.items()}
+
+
+def has_valid_cf_cookies(domain: str) -> bool:
+    """Check if we have valid Cloudflare cookies for a domain."""
+    return bool(get_cf_cookies_for_domain(domain))
+
+
+def get_cf_user_agent_for_domain(domain: str) -> Optional[str]:
+    """Get the User-Agent that was used during bypass for a domain."""
+    if not domain:
+        return None
+    with _cf_cookies_lock:
+        return _cf_user_agents.get(_get_base_domain(domain))
+
+
+def clear_cf_cookies(domain: str = None) -> None:
+    """Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
+    with _cf_cookies_lock:
+        if domain:
+            base_domain = _get_base_domain(domain)
+            _cf_cookies.pop(base_domain, None)
+            _cf_user_agents.pop(base_domain, None)
+        else:
+            _cf_cookies.clear()
+            _cf_user_agents.clear()
+
+
+def _cleanup_orphan_processes() -> int:
+    """Kill orphan Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode."""
+    if not env.DOCKERMODE:
+        return 0
+
+    _stop_ffmpeg_recording()
+
+    processes_to_kill = ["chrome", "chromium", "Xvfb", "ffmpeg"]
+    total_killed = 0
+
+    logger.debug("Checking for orphan processes...")
+    logger.log_resource_usage()
+
+    for proc_name in processes_to_kill:
+        try:
+            result = subprocess.run(
+                ["pgrep", "-f", proc_name],
+                capture_output=True,
+                text=True,
+                timeout=5
+            )
+            if result.returncode != 0 or not result.stdout.strip():
+                continue
+
+            pids = result.stdout.strip().split('\n')
+            count = len(pids)
+            logger.info(f"Found {count} orphan {proc_name} process(es), killing...")
+
+            kill_result = subprocess.run(
+                ["pkill", "-9", "-f", proc_name],
+                capture_output=True,
+                timeout=5
+            )
+            if kill_result.returncode == 0:
+                total_killed += count
+            else:
+                logger.warning(f"pkill for {proc_name} returned {kill_result.returncode}")
+
+        except subprocess.TimeoutExpired:
+            logger.warning(f"Timeout while checking for {proc_name} processes")
+        except Exception as e:
+            logger.debug(f"Error checking for {proc_name} processes: {e}")
+
+    if total_killed > 0:
+        time.sleep(1)
+        logger.info(f"Cleaned up {total_killed} orphan process(es)")
+        logger.log_resource_usage()
+    else:
+        logger.debug("No orphan processes found")
+
+    return total_killed
+
+async def _get_page_info(page) -> tuple[str, str, str]:
+    """Extract page title, body text, and current URL safely."""
+    try:
+        title = (await page.get_title() or "").lower()
+    except Exception:
+        title = ""
+    try:
+        body = await page.evaluate("document.body ? document.body.innerText : ''")
+        body = (body or "").lower()
+    except Exception:
+        body = ""
+    try:
+        current_url = await page.get_current_url() or ""
+    except Exception:
+        current_url = ""
+    return title, body, current_url
+
+
+def _check_indicators(title: str, body: str, indicators: list[str]) -> Optional[str]:
+    """Check if any indicator is present in title or body. Returns the found indicator or None."""
+    for indicator in indicators:
+        if indicator in title or indicator in body:
+            return indicator
+    return None
+
+def _has_cloudflare_patterns(body: str, url: str) -> bool:
+    """Check for Cloudflare-specific patterns in body or URL."""
+    return "cf-" in body or "cloudflare" in url.lower() or "/cdn-cgi/" in url
+
+async def _detect_challenge_type(page) -> str:
+    """Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'."""
+    try:
+        title, body, current_url = await _get_page_info(page)
+
+        # DDOS-Guard indicators
+        if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS):
+            logger.debug(f"DDOS-Guard indicator found: '{found}'")
+            return "ddos_guard"
+
+        # Cloudflare indicators
+        if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS):
+            logger.debug(f"Cloudflare indicator found: '{found}'")
+            return "cloudflare"
+
+        # Check URL patterns
+        if _has_cloudflare_patterns(body, current_url):
+            return "cloudflare"
+
+        return "none"
+    except Exception as e:
+        logger.warning(f"Error detecting challenge type: {e}")
+        return "none"
+
+async def _is_bypassed(page, escape_emojis: bool = True) -> bool:
+    """Check if the protection has been bypassed."""
+    try:
+        title, body, current_url = await _get_page_info(page)
+        body_len = len(body.strip())
+
+        # Long page content = probably bypassed
+        if body_len > 100000:
+            logger.debug(f"Page content too long, probably bypassed (len: {body_len})")
+            return True
+
+        # Multiple emojis = probably real content
+        if escape_emojis:
+            import emoji
+            if len(emoji.emoji_list(body)) >= 3:
+                logger.debug("Detected emojis in page, probably bypassed")
+                return True
+
+        # Check for protection indicators (means NOT bypassed)
+        if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS):
+            return False
+
+        # Cloudflare URL patterns
+        if _has_cloudflare_patterns(body, current_url):
+            logger.debug("Cloudflare patterns detected in page")
+            return False
+
+        # Page too short = still loading
+        if body_len < 50:
+            logger.debug("Page content too short, might still be loading")
+            return False
+
+        logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {body_len}")
+        return True
+
+    except Exception as e:
+        logger.warning(f"Error checking bypass status: {e}")
+        return False
+
+async def _bypass_method_humanlike(page) -> bool:
+    """Human-like behavior with scroll, wait, and reload."""
+    try:
+        logger.debug("Attempting bypass: human-like interaction")
+        await asyncio.sleep(random.uniform(6, 10))
+
+        try:
+            await page.evaluate("window.scrollTo(0, 10000);")
+            await page.wait()
+            await asyncio.sleep(random.uniform(1, 2))
+            await page.evaluate("window.scrollTo(0, 0);")
+            await page.wait()
+            await asyncio.sleep(random.uniform(2, 3))
+        except Exception as e:
+            logger.debug(f"Scroll behavior failed: {e}")
+
+        if await _is_bypassed(page):
+            return True
+
+        logger.debug("Trying page refresh...")
+        await page.reload(ignore_cache=True)
+        await asyncio.sleep(random.uniform(5, 8))
+
+        if await _is_bypassed(page):
+            return True
+
+        try:
+            await page.solve_captcha()
+            await asyncio.sleep(random.uniform(3, 5))
+        except Exception as e:
+            logger.debug(f"Final captcha click failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"Human-like method failed: {e}")
+        return False
+
+
+async def _bypass_method_cdp_solve(page) -> bool:
+    """CDP Mode with solve_captcha() - auto-detects challenge type."""
+    try:
+        logger.debug("Attempting bypass: CDP solve_captcha")
+        await page.solve_captcha()
+        await asyncio.sleep(random.uniform(3, 5))
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP solve_captcha failed: {e}")
+        return False
+
+
+CDP_CLICK_SELECTORS = [
+    "#turnstile-widget div",      # Cloudflare Turnstile
+    "#cf-turnstile div",          # Alternative CF Turnstile
+    "iframe[src*='challenges']",  # CF challenge iframe
+    "input[type='checkbox']",     # Generic checkbox (DDOS-Guard)
+    "[class*='checkbox']",        # Class-based checkbox
+    "#challenge-running",         # CF challenge indicator
+]
+
+
+async def _bypass_method_cdp_click(page) -> bool:
+    """CDP Mode with native clicking - no PyAutoGUI dependency."""
+    try:
+        logger.debug("Attempting bypass: CDP native click")
+
+        for selector in CDP_CLICK_SELECTORS:
+            try:
+                if not await page.is_element_visible(selector):
+                    continue
+
+                logger.debug(f"CDP clicking: {selector}")
+                await page.click(selector)
+                await asyncio.sleep(random.uniform(2, 4))
+
+                if await _is_bypassed(page):
+                    return True
+            except Exception as e:
+                logger.debug(f"CDP click on '{selector}' failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP Mode click failed: {e}")
+        return False
+
+
+CDP_GUI_CLICK_SELECTORS = [
+    "#turnstile-widget div",      # Cloudflare Turnstile
+    "#cf-turnstile div",          # Alternative CF Turnstile
+    "#challenge-stage div",       # CF challenge stage
+    "input[type='checkbox']",     # Generic checkbox
+    "[class*='cb-i']",            # DDOS-Guard checkbox
+]
+
+
+async def _bypass_method_cdp_gui_click(page) -> bool:
+    """CDP Mode with gui_click-style behavior."""
+    try:
+        logger.debug("Attempting bypass: CDP gui_click (mouse-based)")
+
+        try:
+            logger.debug("Trying solve_captcha()")
+            await page.solve_captcha()
+            await asyncio.sleep(random.uniform(3, 5))
+
+            if await _is_bypassed(page):
+                return True
+        except Exception as e:
+            logger.debug(f"solve_captcha() failed: {e}")
+
+        for selector in CDP_GUI_CLICK_SELECTORS:
+            try:
+                if not await page.is_element_visible(selector):
+                    continue
+
+                logger.debug(f"CDP click_with_offset: {selector}")
+                await page.click_with_offset(selector, 0, 0, center=True)
+                await asyncio.sleep(random.uniform(3, 5))
+
+                if await _is_bypassed(page):
+                    return True
+            except Exception as e:
+                logger.debug(f"CDP gui_click on '{selector}' failed: {e}")
+
+        return await _is_bypassed(page)
+    except Exception as e:
+        logger.debug(f"CDP Mode gui_click failed: {e}")
+        return False
+
+
+BYPASS_METHODS = [
+    _bypass_method_cdp_solve,
+    _bypass_method_cdp_gui_click,
+    _bypass_method_cdp_click,
+    _bypass_method_humanlike,
+]
+
+MAX_CONSECUTIVE_SAME_CHALLENGE = 3
+
+
+def _check_cancellation(cancel_flag: Optional[Event], message: str) -> None:
+    """Check if cancellation was requested and raise if so."""
+    if cancel_flag and cancel_flag.is_set():
+        logger.info(message)
+        raise BypassCancelledException("Bypass cancelled")
+
+
+async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional[Event] = None) -> bool:
+    """Attempt to bypass Cloudflare/DDOS-Guard protection using multiple methods."""
+    max_retries = max_retries if max_retries is not None else app_config.MAX_RETRY
+
+    last_challenge_type = None
+    consecutive_same_challenge = 0
+    # Allow at least one full pass through all bypass methods before aborting due to a "stuck" challenge.
+    min_same_challenge_before_abort = max(MAX_CONSECUTIVE_SAME_CHALLENGE, len(BYPASS_METHODS) + 1)
+
+    for try_count in range(max_retries):
+        _check_cancellation(cancel_flag, "Bypass cancelled by user")
+
+        if await _is_bypassed(page):
+            if try_count == 0:
+                logger.info("Page already bypassed")
+            return True
+
+        challenge_type = await _detect_challenge_type(page)
+        logger.debug(f"Challenge detected: {challenge_type}")
+
+        # No challenge detected but page doesn't look bypassed - wait and retry
+        if challenge_type == "none":
+            logger.info("No challenge detected, waiting for page to settle...")
+            await asyncio.sleep(random.uniform(2, 3))
+            if await _is_bypassed(page):
+                return True
+            # Try a simple refresh instead of captcha methods
+            try:
+                await page.reload(ignore_cache=True)
+                await asyncio.sleep(random.uniform(1, 2))
+                if await _is_bypassed(page):
+                    logger.info("Bypass successful after refresh")
+                    return True
+            except Exception as e:
+                logger.debug(f"Refresh during no-challenge wait failed: {e}")
+            continue
+
+        if challenge_type == last_challenge_type:
+            consecutive_same_challenge += 1
+            if consecutive_same_challenge >= min_same_challenge_before_abort:
+                logger.warning(
+                    f"Same challenge ({challenge_type}) detected {consecutive_same_challenge} times - aborting"
+                )
+                return False
+        else:
+            consecutive_same_challenge = 1
+        last_challenge_type = challenge_type
+
+        method = BYPASS_METHODS[try_count % len(BYPASS_METHODS)]
+        logger.info(f"Bypass attempt {try_count + 1}/{max_retries} using {method.__name__}")
+
+        if try_count > 0:
+            wait_time = min(random.uniform(2, 4) * try_count, 12)
+            logger.info(f"Waiting {wait_time:.1f}s before trying...")
+            for _ in range(int(wait_time)):
+                _check_cancellation(cancel_flag, "Bypass cancelled during wait")
+                await asyncio.sleep(1)
+            await asyncio.sleep(wait_time - int(wait_time))
+
+        try:
+            if await method(page):
+                logger.info(f"Bypass successful using {method.__name__}")
+                return True
+        except BypassCancelledException:
+            raise
+        except Exception as e:
+            logger.warning(f"Exception in {method.__name__}: {e}")
+
+        logger.info(f"Bypass method {method.__name__} failed.")
+
+    logger.warning("Exceeded maximum retries. Bypass failed.")
+    return False
+
+def _get_browser_args() -> list[str]:
+    """Build extra Chrome arguments, pre-resolving hostnames via patched DNS.
+
+    Pre-resolves AA hostnames and passes IPs to Chrome via --host-resolver-rules,
+    bypassing Chrome's DNS entirely for those hosts.
+    """
+    arguments = [
+        "--ignore-certificate-errors",
+        "--ignore-ssl-errors",
+        "--allow-running-insecure-content",
+        "--ignore-certificate-errors-spki-list",
+        "--ignore-certificate-errors-skip-list",
+        # Chrome 144+ disabled automatic SwiftShader fallback for WebGL (security reasons).
+        # Without this flag, WebGL is broken in headless/Docker which triggers bot detection.
+        # See: https://issues.chromium.org/issues/40277080
+        "--enable-unsafe-swiftshader",
+    ]
+
+    if app_config.get("DEBUG", False):
+        arguments.extend([
+            "--enable-logging",
+            "--v=1",
+            "--log-file=" + str(LOG_DIR / "chrome_browser.log")
+        ])
+
+    host_rules = _build_host_resolver_rules()
+    if host_rules:
+        arguments.append(f'--host-resolver-rules={", ".join(host_rules)}')
+        logger.debug(f"Chrome: Using host resolver rules for {len(host_rules)} hosts")
+    else:
+        logger.warning("Chrome: No hosts could be pre-resolved")
+
+    return arguments
+
+
+def _build_host_resolver_rules() -> list[str]:
+    """Pre-resolve AA hostnames and build Chrome host resolver rules."""
+    host_rules = []
+
+    try:
+        for url in network.get_available_aa_urls():
+            hostname = urlparse(url).hostname
+            if not hostname:
+                continue
+
+            try:
+                results = socket.getaddrinfo(hostname, 443, socket.AF_INET)
+                if results:
+                    ip = results[0][4][0]
+                    host_rules.append(f"MAP {hostname} {ip}")
+                    logger.debug(f"Chrome: Pre-resolved {hostname} -> {ip}")
+                else:
+                    logger.warning(f"Chrome: No addresses returned for {hostname}")
+            except socket.gaierror as e:
+                logger.warning(f"Chrome: Could not pre-resolve {hostname}: {e}")
+    except Exception as e:
+        logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}")
+
+    return host_rules
+
+DRIVER_RESET_ERRORS = {"ProtocolException", "RuntimeError", "TimeoutError"}
+
+
+async def _get(url: str, driver, cancel_flag: Optional[Event] = None) -> str:
+    """Fetch URL with Cloudflare bypass using a CDP browser."""
+    _check_cancellation(cancel_flag, "Bypass cancelled before starting")
+
+    logger.debug(f"CDP_GET: {url}")
+
+    logger.debug("Opening URL with SeleniumBase CDP...")
+    page = await driver.get(url)
+    try:
+        await page.wait()
+    except Exception:
+        pass
+
+    _check_cancellation(cancel_flag, "Bypass cancelled after page load")
+
+    try:
+        current_url = await page.get_current_url()
+        title = await page.get_title()
+        logger.debug(f"Page loaded - URL: {current_url}, Title: {title}")
+    except Exception as e:
+        logger.debug(f"Could not get page info: {e}")
+
+    logger.debug("Starting bypass process...")
+    if await _bypass(page, cancel_flag=cancel_flag):
+        await _extract_cookies_from_cdp(driver, page, url)
+        return await page.get_page_source()
+
+    logger.warning("Bypass completed but page still shows protection")
+    try:
+        body = await page.evaluate("document.body ? document.body.innerText : ''")
+        if body:
+            logger.debug(f"Page content: {body[:500]}..." if len(body) > 500 else body)
+    except Exception:
+        pass
+
+    return ""
+
+
+def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = None) -> str:
+    """Fetch a URL with protection bypass. Creates fresh Chrome instance for each bypass."""
+    retry = retry if retry is not None else app_config.MAX_RETRY
+
+    with LOCKED:
+        # Try cookies first - another request may have completed bypass while waiting
+        cached_result = _try_with_cached_cookies(url, urlparse(url).hostname or "")
+        if cached_result:
+            return cached_result
+
+        async def _run_bypass() -> str:
+            driver = None
+            try:
+                driver = await _create_cdp_browser(url)
+
+                for attempt in range(retry):
+                    _check_cancellation(cancel_flag, "Bypass cancelled before attempt")
+
+                    try:
+                        result = await _get(url, driver, cancel_flag)
+                        if result:
+                            return result
+                    except BypassCancelledException:
+                        raise
+                    except Exception as e:
+                        error_details = f"{type(e).__name__}: {e}"
+                        logger.warning(f"Bypass failed (attempt {attempt + 1}/{retry}): {error_details}")
+                        logger.debug(f"Stack trace: {traceback.format_exc()}")
+
+                        # On CDP errors, quit and create a fresh browser
+                        if type(e).__name__ in DRIVER_RESET_ERRORS:
+                            logger.info("Restarting Chrome due to browser error...")
+                            await _close_cdp_driver(driver)
+                            driver = await _create_cdp_browser(url)
+
+                logger.error(f"Bypass failed after {retry} attempts")
+                return ""
+            finally:
+                if driver:
+                    await _close_cdp_driver(driver)
+
+        return _CDP_WORKER.run(_run_bypass())
+
+def _get_proxy_string(url: str) -> Optional[str]:
+    """Return a single proxy string for CDP, honoring NO_PROXY."""
+    proxies = get_proxies(url)
+    if not proxies:
+        return None
+    proxy_url = proxies.get("https") or proxies.get("http")
+    return proxy_url or None
+
+
+async def _create_cdp_browser(url: str) -> Any:
+    """Create a fresh CDP browser instance."""
+    browser_args = _get_browser_args()
+    screen_width, screen_height = get_screen_size()
+    display_width = screen_width + 100
+    display_height = screen_height + 150
+    proxy = _get_proxy_string(url)
+
+    logger.debug(f"Creating Pure CDP browser with args: {browser_args}")
+    logger.debug(f"Browser screen size: {screen_width}x{screen_height}")
+
+    driver = await cdp_driver.start_async(
+        headless=False,
+        headed=False,
+        xvfb=True,
+        xvfb_metrics=f"{display_width},{display_height}",
+        sandbox=False,
+        lang="en",
+        incognito=True,
+        ad_block=True,
+        proxy=proxy,
+        browser_args=browser_args,
+    )
+
+    try:
+        await driver.page.set_window_rect(0, 0, screen_width, screen_height)
+    except Exception as e:
+        logger.debug(f"Failed to set window size: {e}")
+
+    # Start FFmpeg recording if debug mode (record each bypass session)
+    if app_config.get("DEBUG", False) and not DISPLAY.get("ffmpeg"):
+        _start_ffmpeg_recording(display=os.environ.get("DISPLAY", ":0"))
+
+    await asyncio.sleep(app_config.DEFAULT_SLEEP)
+    logger.info("Chrome browser ready (Pure CDP)")
+    logger.log_resource_usage()
+    return driver
+
+
+async def _close_cdp_driver(driver) -> None:
+    """Close CDP connections and stop the browser."""
+    if not driver:
+        return
+
+    logger.debug("Quitting Chrome browser (CDP)...")
+
+    _stop_ffmpeg_recording()
+
+    try:
+        connections = []
+        if hasattr(driver, "connection") and driver.connection:
+            connections.append(driver.connection)
+        if hasattr(driver, "targets") and driver.targets:
+            connections.extend(driver.targets)
+        for conn in connections:
+            try:
+                await conn.aclose()
+            except Exception as e:
+                logger.debug(f"Failed to close websocket connection: {e}")
+    except Exception as e:
+        logger.debug(f"Error during connection cleanup: {e}")
+
+    try:
+        driver.stop()
+        logger.debug("Stopped CDP browser")
+    except Exception as e:
+        logger.debug(f"CDP stop: {e}")
+
+    if env.DOCKERMODE:
+        await asyncio.sleep(0.3)
+        try:
+            pid = getattr(driver, "_process_pid", None)
+
+            def _pid_alive(check_pid: int) -> bool:
+                try:
+                    os.kill(check_pid, 0)
+                except ProcessLookupError:
+                    return False
+                except PermissionError:
+                    return True
+                return True
+
+            if pid and _pid_alive(pid):
+                try:
+                    os.kill(pid, signal.SIGTERM)
+                    await asyncio.sleep(0.1)
+                    if _pid_alive(pid):
+                        os.kill(pid, signal.SIGKILL)
+                    logger.debug(f"Killed Chrome pid {pid}")
+                except Exception as e:
+                    logger.debug(f"Failed to kill Chrome pid {pid}: {e}")
+        except Exception as e:
+            logger.debug(f"Process cleanup failed: {e}")
+
+    logger.log_resource_usage()
+
+
+def _start_ffmpeg_recording(display: str) -> None:
+    """Start FFmpeg screen recording for debug mode."""
+    global DISPLAY
+    RECORDING_DIR.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
+    output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
+
+    screen_width, screen_height = get_screen_size()
+    display_width = screen_width + 100
+    display_height = screen_height + 150
+
+    ffmpeg_cmd = [
+        "ffmpeg", "-y", "-f", "x11grab",
+        "-video_size", f"{display_width}x{display_height}",
+        "-i", display,
+        "-c:v", "libx264", "-preset", "ultrafast",
+        "-maxrate", "700k", "-bufsize", "1400k", "-crf", "36",
+        "-pix_fmt", "yuv420p", "-tune", "animation",
+        "-x264-params", "bframes=0:deblock=-1,-1",
+        "-r", "15", "-an",
+        output_file.as_posix(),
+        "-nostats", "-loglevel", "0"
+    ]
+    logger.debug("Starting FFmpeg recording to %s", output_file)
+    logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
+    DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
+    DISPLAY["ffmpeg_output"] = output_file
+
+
+def _stop_ffmpeg_recording() -> None:
+    """Stop FFmpeg screen recording if running."""
+    import signal
+    global DISPLAY
+    proc = DISPLAY.get("ffmpeg")
+    output_file = DISPLAY.get("ffmpeg_output")
+    if not proc:
+        return
+    if proc.poll() is not None:
+        logger.debug("FFmpeg already stopped")
+        DISPLAY["ffmpeg"] = None
+        DISPLAY["ffmpeg_output"] = None
+        return
+    try:
+        proc.send_signal(signal.SIGINT)
+        proc.wait(timeout=5)
+        logger.debug("Stopped ffmpeg recording")
+    except Exception as e:
+        logger.debug(f"ffmpeg stop: {e}")
+        try:
+            proc.terminate()
+            proc.wait(timeout=2)
+        except Exception:
+            pass
+        try:
+            proc.kill()
+        except Exception:
+            pass
+    DISPLAY["ffmpeg"] = None
+    DISPLAY["ffmpeg_output"] = None
+
+
+def _try_with_cached_cookies(url: str, hostname: str) -> Optional[str]:
+    """Attempt request with cached cookies before using Chrome."""
+    cookies = get_cf_cookies_for_domain(hostname)
+    if not cookies:
+        return None
+
+    try:
+        headers = {}
+        stored_ua = get_cf_user_agent_for_domain(hostname)
+        if stored_ua:
+            headers['User-Agent'] = stored_ua
+
+        logger.debug(f"Trying request with cached cookies: {url}")
+        response = requests.get(url, cookies=cookies, headers=headers, proxies=get_proxies(url), timeout=(5, 10))
+        if response.status_code == 200:
+            logger.debug("Cached cookies worked, skipped Chrome bypass")
+            return response.text
+    except Exception:
+        pass
+
+    return None
+
+
+def get_bypassed_page(
+    url: str,
+    selector: Optional[network.AAMirrorSelector] = None,
+    cancel_flag: Optional[Event] = None
+) -> Optional[str]:
+    """Fetch HTML content from a URL using the internal Cloudflare Bypasser."""
+    sel = selector or network.AAMirrorSelector()
+    attempt_url = sel.rewrite(url)
+    hostname = urlparse(attempt_url).hostname or ""
+
+    cached_result = _try_with_cached_cookies(attempt_url, hostname)
+    if cached_result:
+        return cached_result
+
+    try:
+        response_html = get(attempt_url, cancel_flag=cancel_flag)
+    except BypassCancelledException:
+        raise
+    except Exception:
+        _check_cancellation(cancel_flag, "Bypass cancelled")
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            attempt_url = sel.rewrite(url)
+            response_html = get(attempt_url, cancel_flag=cancel_flag)
+        else:
+            raise
+
+    if not response_html.strip():
+        raise requests.exceptions.RequestException("Failed to bypass Cloudflare")
+
+    return response_html

shelfmark/config/booklore_settings.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.core.logger import setup_logger
+from shelfmark.download.outputs.booklore import (
+    BookloreConfig,
+    BookloreError,
+    booklore_list_libraries,
+    booklore_login,
+)
+
+logger = setup_logger(__name__)
+
+_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
+    "key": None,
+    "library_options": [],
+    "path_options": [],
+}
+
+
+def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
+    return f"{base_url}|{username}|{hash(password)}"
+
+
+def _get_booklore_select_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    # library_id/path_id are not used for login/library listing
+    booklore_config = BookloreConfig(
+        base_url=base_url.rstrip("/"),
+        username=username,
+        password=password,
+        library_id=1,
+        path_id=1,
+        verify_tls=True,
+        refresh_after_upload=True,
+    )
+
+    token = booklore_login(booklore_config)
+    libraries = booklore_list_libraries(booklore_config, token) or []
+    logger.debug("Booklore libraries response: %s", libraries)
+
+    library_options: list[dict[str, Any]] = []
+    path_options: list[dict[str, Any]] = []
+
+    for library in libraries:
+        if not isinstance(library, dict):
+            continue
+
+        library_id = library.get("id")
+        if library_id is None:
+            continue
+
+        library_name = str(library.get("name") or f"Library {library_id}")
+        library_id_str = str(library_id)
+
+        library_options.append({"value": library_id_str, "label": library_name})
+
+        paths = library.get("paths") or []
+        if not isinstance(paths, list):
+            continue
+
+        for path in paths:
+            if not isinstance(path, dict):
+                continue
+
+            path_id = path.get("id")
+            if path_id is None:
+                continue
+
+            path_label = str(path.get("path") or f"Path {path_id}")
+            path_options.append(
+                {
+                    "value": str(path_id),
+                    "label": f"{library_name}: {path_label}",
+                    "childOf": library_id_str,
+                }
+            )
+
+    logger.debug(
+        "Booklore options built: libraries=%d paths=%d",
+        len(library_options),
+        len(path_options),
+    )
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    _BOOKLORE_OPTIONS_CACHE.update(
+        {
+            "key": cache_key,
+            "library_options": library_options,
+            "path_options": path_options,
+        }
+    )
+
+    return library_options, path_options
+
+
+def _get_booklore_cached_options(
+    base_url: str,
+    username: str,
+    password: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+    if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+        return (
+            _BOOKLORE_OPTIONS_CACHE.get("library_options", []),
+            _BOOKLORE_OPTIONS_CACHE.get("path_options", []),
+        )
+
+    return _get_booklore_select_options(base_url, username, password)
+
+
+def get_booklore_library_options() -> list[dict[str, Any]]:
+    """Build Booklore library options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        library_options, _ = _get_booklore_cached_options(base_url, username, password)
+        return library_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore libraries: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
+        return []
+
+
+def get_booklore_path_options() -> list[dict[str, Any]]:
+    """Build Booklore path options dynamically from config."""
+    if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
+        return []
+
+    base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
+    password = config.get("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url or not username or not password:
+        return []
+
+    cache_key = _get_booklore_cache_key(base_url, username, password)
+
+    try:
+        _, path_options = _get_booklore_cached_options(base_url, username, password)
+        return path_options
+    except Exception as exc:
+        logger.error(f"Failed to fetch Booklore paths: {exc}")
+        if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
+            return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
+        return []
+
+
+def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test the Booklore connection using current form values."""
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
+    username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
+    password = _get_value("BOOKLORE_PASSWORD", "") or ""
+
+    if not base_url:
+        return {"success": False, "message": "Booklore URL is required"}
+    if not username:
+        return {"success": False, "message": "Booklore username is required"}
+    if not password:
+        return {"success": False, "message": "Booklore password is required"}
+
+    try:
+        library_options, _ = _get_booklore_select_options(base_url, username, password)
+
+        message = "Connected to Booklore"
+        if library_options:
+            message = f"Connected to Booklore ({len(library_options)} libraries)"
+
+        return {"success": True, "message": message}
+    except BookloreError as exc:
+        return {"success": False, "message": str(exc)}

shelfmark/config/email_settings.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Any
+
+from shelfmark.core.config import config
+from shelfmark.download.outputs.email import EmailOutputError, build_email_smtp_config, test_smtp_connection
+
+
+def test_email_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Test SMTP connectivity using current form values (including unsaved changes)."""
+
+    current_values = current_values or {}
+
+    def _get_value(key: str, default: Any = None) -> Any:
+        value = current_values.get(key)
+        if value not in (None, ""):
+            return value
+        if default is None:
+            return config.get(key)
+        return config.get(key, default)
+
+    settings = {
+        "EMAIL_SMTP_HOST": _get_value("EMAIL_SMTP_HOST", ""),
+        "EMAIL_SMTP_PORT": _get_value("EMAIL_SMTP_PORT", 587),
+        "EMAIL_SMTP_SECURITY": _get_value("EMAIL_SMTP_SECURITY", "starttls"),
+        "EMAIL_SMTP_USERNAME": _get_value("EMAIL_SMTP_USERNAME", ""),
+        "EMAIL_SMTP_PASSWORD": _get_value("EMAIL_SMTP_PASSWORD", ""),
+        "EMAIL_FROM": _get_value("EMAIL_FROM", ""),
+        "EMAIL_SUBJECT_TEMPLATE": _get_value("EMAIL_SUBJECT_TEMPLATE", "{Title}"),
+        "EMAIL_SMTP_TIMEOUT_SECONDS": _get_value("EMAIL_SMTP_TIMEOUT_SECONDS", 60),
+        "EMAIL_ALLOW_UNVERIFIED_TLS": _get_value("EMAIL_ALLOW_UNVERIFIED_TLS", False),
+    }
+
+    try:
+        smtp_config = build_email_smtp_config(settings)
+        test_smtp_connection(smtp_config)
+        return {"success": True, "message": "Connected to SMTP server"}
+    except EmailOutputError as exc:
+        return {"success": False, "message": str(exc)}
+    except Exception as exc:
+        return {"success": False, "message": f"SMTP test failed: {exc}"}
+

shelfmark/config/env.py

@@ -0,0 +1,162 @@
+"""Bootstrap environment variables. No local dependencies - import first."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+
+def string_to_bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ["true", "yes", "1", "y"]
+
+
+def _read_debug_from_config() -> bool:
+    """Read DEBUG from env var or config file (import-time safe)."""
+    env_debug = os.environ.get("DEBUG")
+    if env_debug is not None:
+        return string_to_bool(env_debug)
+
+    # Try to read from config file
+    config_dir = Path(os.getenv("CONFIG_DIR", "/config"))
+    config_file = config_dir / "plugins" / "advanced.json"
+
+    if config_file.exists():
+        try:
+            with open(config_file, "r") as f:
+                config = json.load(f)
+                if "DEBUG" in config:
+                    return bool(config["DEBUG"])
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return False
+
+
+def _is_sqlite_file(path: Path) -> bool:
+    """Check if a file is a valid SQLite database by reading magic bytes."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(16)
+            return header[:16] == b"SQLite format 3\x00"
+    except (OSError, PermissionError):
+        return False
+
+
+def _resolve_cwa_db_path() -> Path | None:
+    """Resolve CWA database path from env var or default location."""
+    env_path = os.getenv("CWA_DB_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file() and _is_sqlite_file(path):
+            return path
+
+    # Check default mount path
+    default_path = Path("/auth/app.db")
+    if default_path.exists() and default_path.is_file() and _is_sqlite_file(default_path):
+        return default_path
+
+    return None
+
+
+def _is_config_dir_writable() -> bool:
+    """Check if the config directory exists and is writable."""
+    try:
+        if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir():
+            return False
+        test_file = CONFIG_DIR / ".write_test"
+        test_file.touch()
+        test_file.unlink()
+        return True
+    except (OSError, PermissionError):
+        return False
+
+
+def is_covers_cache_enabled() -> bool:
+    """Check if cover caching is enabled (requires setting + writable config dir)."""
+    from shelfmark.core.config import config
+    setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
+    return setting_enabled and _is_config_dir_writable()
+
+
+# =============================================================================
+# Bootstrap paths - needed before settings registry is available
+# =============================================================================
+
+CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
+LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
+LOG_DIR = LOG_ROOT / "shelfmark"
+LOG_FILE = LOG_DIR / "shelfmark.log"
+TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
+INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
+
+
+# =============================================================================
+# Logger configuration - needed before settings registry is available
+# =============================================================================
+
+DEBUG = _read_debug_from_config()
+LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
+ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
+
+
+# =============================================================================
+# Flask configuration - needed before app starts
+# =============================================================================
+
+FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
+
+
+# =============================================================================
+# Authentication
+# =============================================================================
+
+SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
+SESSION_COOKIE_NAME = "shelfmark_session"
+CWA_DB_PATH = _resolve_cwa_db_path()
+
+
+# =============================================================================
+# Version information from Docker build
+# =============================================================================
+
+BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
+RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
+
+
+# =============================================================================
+# Capability detection - runtime checks, not user-configurable
+# =============================================================================
+
+DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
+TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
+USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
+
+
+# =============================================================================
+# Onboarding
+# =============================================================================
+
+# Set to false to skip the onboarding wizard entirely (useful for ephemeral storage)
+ONBOARDING = string_to_bool(os.getenv("ONBOARDING", "true"))
+
+
+# =============================================================================
+# Debug/development settings
+# =============================================================================
+
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
+
+
+# =============================================================================
+# Legacy migration support - will be removed in future version
+# =============================================================================
+
+# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
+# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
+_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))

shelfmark/config/migrations.py

@@ -0,0 +1,123 @@
+"""Configuration migration helpers."""
+
+import json
+from typing import Any, Callable
+
+
+_DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
+    "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
+    "CWA_RESTRICT_SETTINGS_TO_ADMIN",
+    "RESTRICT_SETTINGS_TO_ADMIN",
+)
+
+
+def _as_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+    return bool(value)
+
+
+def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None:
+    """Pick the best legacy admin-restriction value to migrate."""
+    auth_method = str(config.get("AUTH_METHOD", "")).strip().lower()
+
+    if (
+        auth_method == "proxy"
+        and "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config
+    ):
+        return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if auth_method == "cwa" and "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    if "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
+        return _as_bool(config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN"))
+
+    return None
+
+
+def migrate_security_settings(
+    *,
+    load_security_config: Callable[[], dict[str, Any]],
+    load_users_config: Callable[[], dict[str, Any]],
+    save_users_config: Callable[[dict[str, Any]], None],
+    ensure_config_dir: Callable[[], None],
+    get_config_path: Callable[[], Any],
+    sync_builtin_admin_user: Callable[[str, str], None],
+    logger: Any,
+) -> None:
+    """Migrate legacy security keys and sync builtin admin credentials."""
+    try:
+        config = load_security_config()
+        users_config = load_users_config()
+        migrated_security = False
+        migrated_users = False
+
+        if "USE_CWA_AUTH" in config:
+            old_value = config.pop("USE_CWA_AUTH")
+            if "AUTH_METHOD" not in config:
+                if old_value:
+                    config["AUTH_METHOD"] = "cwa"
+                    logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
+                else:
+                    if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
+                        config["AUTH_METHOD"] = "builtin"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
+                    else:
+                        config["AUTH_METHOD"] = "none"
+                        logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
+                migrated_security = True
+            else:
+                logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
+                migrated_security = True
+
+        if "RESTRICT_SETTINGS_TO_ADMIN" not in users_config:
+            legacy_restrict = _pick_legacy_settings_restriction(config)
+            if legacy_restrict is not None:
+                save_users_config({"RESTRICT_SETTINGS_TO_ADMIN": legacy_restrict})
+                migrated_users = True
+                logger.info(
+                    "Migrated legacy settings-admin restriction to users.RESTRICT_SETTINGS_TO_ADMIN="
+                    f"{legacy_restrict}"
+                )
+
+        for deprecated_key in _DEPRECATED_SETTINGS_RESTRICTION_KEYS:
+            if deprecated_key in config:
+                config.pop(deprecated_key, None)
+                migrated_security = True
+                logger.info(f"Removed deprecated security setting: {deprecated_key}")
+
+        try:
+            sync_builtin_admin_user(
+                config.get("BUILTIN_USERNAME", ""),
+                config.get("BUILTIN_PASSWORD_HASH", ""),
+            )
+        except Exception as exc:
+            logger.error(
+                "Failed to sync builtin credentials to users database during migration: "
+                f"{exc}"
+            )
+
+        if migrated_security:
+            ensure_config_dir()
+            config_path = get_config_path()
+            with open(config_path, "w") as f:
+                json.dump(config, f, indent=2)
+            logger.info("Security settings migration completed successfully")
+        elif migrated_users:
+            logger.info("Users settings migration completed successfully")
+        else:
+            logger.debug("No security settings migration needed")
+
+    except FileNotFoundError:
+        logger.debug("No existing security config file found - nothing to migrate")
+    except Exception as exc:
+        logger.error(f"Failed to migrate security settings: {exc}")

shelfmark/config/notifications_settings.py

@@ -0,0 +1,337 @@
+"""Notifications settings tab registration."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlsplit
+
+from shelfmark.core.notifications import NotificationEvent, send_test_notification
+from shelfmark.core.settings_registry import (
+    ActionButton,
+    HeadingField,
+    TableField,
+    load_config_file,
+    register_on_save,
+    register_settings,
+)
+
+_URL_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*$")
+
+_ROUTE_EVENT_ALL = "all"
+_ADMIN_EVENT_OPTIONS = [
+    {"value": NotificationEvent.REQUEST_CREATED.value, "label": "New request submitted"},
+    {"value": NotificationEvent.REQUEST_FULFILLED.value, "label": "Request approved"},
+    {"value": NotificationEvent.REQUEST_REJECTED.value, "label": "Request rejected"},
+    {"value": NotificationEvent.DOWNLOAD_COMPLETE.value, "label": "Download complete"},
+    {"value": NotificationEvent.DOWNLOAD_FAILED.value, "label": "Download failed"},
+]
+_ROUTE_EVENT_OPTIONS = [
+    {"value": _ROUTE_EVENT_ALL, "label": "All"},
+    *_ADMIN_EVENT_OPTIONS,
+]
+_ROUTE_EVENT_ORDER = [option["value"] for option in _ROUTE_EVENT_OPTIONS]
+_ROUTE_EVENT_INDEX = {event: index for index, event in enumerate(_ROUTE_EVENT_ORDER)}
+_ALLOWED_ROUTE_EVENTS = set(_ROUTE_EVENT_ORDER)
+
+_DEFAULT_ROUTE_ROWS = [{"event": [_ROUTE_EVENT_ALL], "url": ""}]
+
+
+def _looks_like_apprise_url(url: str) -> bool:
+    split = urlsplit(url)
+    if not split.scheme:
+        return False
+    if not _URL_SCHEME_RE.match(split.scheme):
+        return False
+    return " " not in url
+
+
+def _coerce_route_rows(value: Any) -> list[dict[str, Any]]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [row for row in value if isinstance(row, dict)]
+    if isinstance(value, dict):
+        return [value]
+    return []
+
+
+def _coerce_route_event_values(value: Any) -> list[Any]:
+    if isinstance(value, list):
+        return value
+    if isinstance(value, (tuple, set)):
+        return list(value)
+    return [value]
+
+
+def _normalize_route_events(value: Any) -> list[str]:
+    normalized: list[str] = []
+    seen: set[str] = set()
+
+    for raw_event in _coerce_route_event_values(value):
+        event = str(raw_event or "").strip().lower()
+        if not event or event not in _ALLOWED_ROUTE_EVENTS:
+            continue
+        if event in seen:
+            continue
+        seen.add(event)
+        normalized.append(event)
+
+    if _ROUTE_EVENT_ALL in seen:
+        return [_ROUTE_EVENT_ALL]
+
+    return sorted(normalized, key=lambda event: _ROUTE_EVENT_INDEX[event])
+
+
+def _normalize_routes(value: Any) -> list[dict[str, Any]]:
+    normalized: list[dict[str, Any]] = []
+    seen: set[tuple[tuple[str, ...], str]] = set()
+
+    for row in _coerce_route_rows(value):
+        events = _normalize_route_events(row.get("event"))
+        if not events:
+            continue
+
+        url = str(row.get("url") or "").strip()
+        key = (tuple(events), url)
+        if key in seen:
+            continue
+        seen.add(key)
+
+        normalized.append({"event": events, "url": url})
+
+    return normalized
+
+
+def _count_invalid_route_events(value: Any) -> int:
+    invalid = 0
+    for row in _coerce_route_rows(value):
+        raw_events = _coerce_route_event_values(row.get("event"))
+        if not raw_events:
+            invalid += 1
+            continue
+
+        for raw_event in raw_events:
+            event = str(raw_event or "").strip().lower()
+            if not event or event not in _ALLOWED_ROUTE_EVENTS:
+                invalid += 1
+    return invalid
+
+
+def _count_invalid_route_urls(routes: list[dict[str, Any]]) -> int:
+    return sum(1 for row in routes if row["url"] and not _looks_like_apprise_url(row["url"]))
+
+
+def _ensure_default_route_row(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return routes if routes else [dict(row) for row in _DEFAULT_ROUTE_ROWS]
+
+
+def _extract_unique_route_urls(routes: list[dict[str, Any]]) -> list[str]:
+    urls: list[str] = []
+    seen: set[str] = set()
+    for row in routes:
+        url = row.get("url", "")
+        if not url:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        urls.append(url)
+    return urls
+
+
+def build_notification_test_result(routes_input: Any, *, scope_label: str) -> dict[str, Any]:
+    invalid_event_count = _count_invalid_route_events(routes_input)
+    if invalid_event_count:
+        return {
+            "success": False,
+            "message": (
+                f"Found {invalid_event_count} invalid {scope_label} notification route event value(s). "
+                "Fix route events before running a test."
+            ),
+        }
+
+    normalized_routes = _normalize_routes(routes_input)
+    invalid_url_count = _count_invalid_route_urls(normalized_routes)
+    if invalid_url_count:
+        return {
+            "success": False,
+            "message": (
+                f"Found {invalid_url_count} invalid {scope_label} notification URL(s). "
+                "Fix route URLs before running a test."
+            ),
+        }
+
+    urls = _extract_unique_route_urls(normalized_routes)
+    if not urls:
+        return {
+            "success": False,
+            "message": f"Add at least one {scope_label} notification URL route first.",
+        }
+
+    return send_test_notification(urls)
+
+
+def normalize_notification_routes(value: Any) -> list[dict[str, Any]]:
+    """Normalize route table rows for notification preferences."""
+    return _normalize_routes(value)
+
+
+def is_valid_notification_url(url: str) -> bool:
+    """Shared URL validation for notifications preferences."""
+    return _looks_like_apprise_url(url)
+
+
+def _on_save_notifications(values: dict[str, Any]) -> dict[str, Any]:
+    existing = load_config_file("notifications")
+    effective: dict[str, Any] = dict(existing)
+    effective.update(values)
+
+    admin_routes_input = effective.get("ADMIN_NOTIFICATION_ROUTES", [])
+    invalid_admin_event_count = _count_invalid_route_events(admin_routes_input)
+    if invalid_admin_event_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_admin_event_count} invalid global notification route event value(s)."
+            ),
+            "values": values,
+        }
+
+    normalized_admin_routes = _normalize_routes(admin_routes_input)
+    invalid_admin_url_count = _count_invalid_route_urls(normalized_admin_routes)
+    if invalid_admin_url_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_admin_url_count} invalid global notification URL(s). "
+                "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+            ),
+            "values": values,
+        }
+
+    user_routes_input = effective.get("USER_NOTIFICATION_ROUTES", [])
+    invalid_user_event_count = _count_invalid_route_events(user_routes_input)
+    if invalid_user_event_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_user_event_count} invalid personal notification route event value(s)."
+            ),
+            "values": values,
+        }
+
+    normalized_user_routes = _normalize_routes(user_routes_input)
+    invalid_user_url_count = _count_invalid_route_urls(normalized_user_routes)
+    if invalid_user_url_count:
+        return {
+            "error": True,
+            "message": (
+                f"Found {invalid_user_url_count} invalid personal notification URL(s). "
+                "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+            ),
+            "values": values,
+        }
+
+    admin_routes_touched = "ADMIN_NOTIFICATION_ROUTES" in values
+    if admin_routes_touched:
+        values["ADMIN_NOTIFICATION_ROUTES"] = _ensure_default_route_row(normalized_admin_routes)
+
+    user_routes_touched = "USER_NOTIFICATION_ROUTES" in values
+    if user_routes_touched:
+        values["USER_NOTIFICATION_ROUTES"] = _ensure_default_route_row(normalized_user_routes)
+
+    return {"error": False, "values": values}
+
+
+def _test_admin_notification_action(current_values: dict[str, Any]) -> dict[str, Any]:
+    persisted = load_config_file("notifications")
+    effective: dict[str, Any] = dict(persisted)
+    if isinstance(current_values, dict):
+        effective.update(current_values)
+
+    routes_input = effective.get("ADMIN_NOTIFICATION_ROUTES", [])
+    return build_notification_test_result(routes_input, scope_label="global")
+
+
+register_on_save("notifications", _on_save_notifications)
+
+
+@register_settings("notifications", "Notifications", icon="bell", order=7)
+def notifications_settings():
+    """Global notifications settings."""
+    return [
+        HeadingField(
+            key="notifications_heading",
+            title="Global Notifications",
+            description=(
+                "Global notifications send selected events for all users to configured routes. "
+                "Users can manage personal notifications in User Preferences."
+            ),
+        ),
+        TableField(
+            key="ADMIN_NOTIFICATION_ROUTES",
+            label="",
+            description=(
+                "Create one route per URL. Start with All, then add event-specific routes "
+                "for targeted delivery. Need format examples? "
+                "[View Apprise URL formats](https://appriseit.com/services/)."
+            ),
+            columns=[
+                {
+                    "key": "event",
+                    "label": "Event",
+                    "type": "multiselect",
+                    "options": _ROUTE_EVENT_OPTIONS,
+                    "defaultValue": [_ROUTE_EVENT_ALL],
+                    "placeholder": "Select events...",
+                },
+                {
+                    "key": "url",
+                    "label": "Notification URL",
+                    "type": "text",
+                    "placeholder": "e.g. ntfys://ntfy.sh/shelfmark",
+                },
+            ],
+            default=[dict(row) for row in _DEFAULT_ROUTE_ROWS],
+            add_label="Add Route",
+            empty_message="No routes configured.",
+        ),
+        ActionButton(
+            key="test_admin_notification",
+            label="Test Notification",
+            description="Send a test notification to all configured global route URLs.",
+            style="primary",
+            callback=_test_admin_notification_action,
+        ),
+        TableField(
+            key="USER_NOTIFICATION_ROUTES",
+            label="",
+            description=(
+                "Create one route per URL. Start with All, then add event-specific routes "
+                "for targeted delivery. Need format examples? "
+                "[View Apprise URL formats](https://appriseit.com/services/)."
+            ),
+            columns=[
+                {
+                    "key": "event",
+                    "label": "Event",
+                    "type": "multiselect",
+                    "options": _ROUTE_EVENT_OPTIONS,
+                    "defaultValue": [_ROUTE_EVENT_ALL],
+                    "placeholder": "Select events...",
+                },
+                {
+                    "key": "url",
+                    "label": "Notification URL",
+                    "type": "text",
+                    "placeholder": "e.g. ntfys://ntfy.sh/username-topic",
+                },
+            ],
+            default=[dict(row) for row in _DEFAULT_ROUTE_ROWS],
+            add_label="Add Route",
+            empty_message="No routes configured.",
+            user_overridable=True,
+            hidden_in_ui=True,
+        ),
+    ]

shelfmark/config/security.py

@@ -0,0 +1,245 @@
+"""Authentication settings registration."""
+
+from typing import Any, Dict, Callable
+
+from shelfmark.config.migrations import migrate_security_settings
+from shelfmark.config.security_handlers import (
+    on_save_security,
+    test_oidc_connection,
+)
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    register_settings,
+    register_on_save,
+    load_config_file,
+    TextField,
+    SelectField,
+    PasswordField,
+    CheckboxField,
+    ActionButton,
+    TagListField,
+)
+from shelfmark.core.user_db import sync_builtin_admin_user
+
+logger = setup_logger(__name__)
+
+
+def _auth_condition(auth_method: str) -> dict[str, str]:
+    return {"field": "AUTH_METHOD", "value": auth_method}
+
+
+def _ui_field(factory: Callable[..., Any], **kwargs: Any) -> Any:
+    return factory(env_supported=False, **kwargs)
+
+
+def _auth_ui_field(factory: Callable[..., Any], auth_method: str, **kwargs: Any) -> Any:
+    return _ui_field(factory, show_when=_auth_condition(auth_method), **kwargs)
+
+
+def _migrate_security_settings() -> None:
+    from shelfmark.core.settings_registry import (
+        _get_config_file_path,
+        _ensure_config_dir,
+        save_config_file,
+    )
+
+    migrate_security_settings(
+        load_security_config=lambda: load_config_file("security"),
+        load_users_config=lambda: load_config_file("users"),
+        save_users_config=lambda values: save_config_file("users", values),
+        ensure_config_dir=lambda: _ensure_config_dir("security"),
+        get_config_path=lambda: _get_config_file_path("security"),
+        sync_builtin_admin_user=sync_builtin_admin_user,
+        logger=logger,
+    )
+
+
+
+def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
+    return on_save_security(values)
+
+
+def _test_oidc_connection() -> Dict[str, Any]:
+    return test_oidc_connection(
+        load_security_config=lambda: load_config_file("security"),
+        logger=logger,
+    )
+
+
+@register_settings("security", "Security", icon="shield", order=5)
+def security_settings():
+    """Security and authentication settings."""
+    from shelfmark.config.env import CWA_DB_PATH
+
+    cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
+
+    auth_method_options = [
+        {"label": "No Authentication", "value": "none"},
+        {"label": "Local", "value": "builtin"},
+        {"label": "Proxy Authentication", "value": "proxy"},
+        {"label": "OIDC (OpenID Connect)", "value": "oidc"},
+    ]
+    if cwa_db_available:
+        auth_method_options.append({"label": "Calibre-Web Database", "value": "cwa"})
+
+    auth_method_description = "Select the authentication method for accessing Shelfmark."
+    if not cwa_db_available:
+        auth_method_description += " Calibre-Web database option requires mounting your Calibre-Web app.db to /auth/app.db."
+
+    fields = [
+        SelectField(
+            key="AUTH_METHOD",
+            label="Authentication Method",
+            description=auth_method_description,
+            options=auth_method_options,
+            default="none",
+            env_supported=False,
+        ),
+        ActionButton(
+            key="open_users_tab",
+            label="Go to Users",
+            description="Configure local users and admin access in the Users tab.",
+            style="primary",
+            show_when=_auth_condition("builtin"),
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_USER_HEADER",
+            label="Proxy Auth User Header",
+            description="The HTTP header your proxy uses to pass the authenticated username.",
+            placeholder="e.g. X-Auth-User",
+            default="X-Auth-User",
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_LOGOUT_URL",
+            label="Proxy Auth Logout URL",
+            description="The URL to redirect users to for logging out. Leave empty to disable logout functionality.",
+            placeholder="https://myauth.example.com/logout",
+            default="",
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_ADMIN_GROUP_HEADER",
+            label="Proxy Auth Admin Group Header",
+            description="Optional: header your proxy uses to pass user groups/roles.",
+            placeholder="e.g. X-Auth-Groups",
+            default="X-Auth-Groups",
+        ),
+        _auth_ui_field(
+            TextField,
+            "proxy",
+            key="PROXY_AUTH_ADMIN_GROUP_NAME",
+            label="Proxy Auth Admin Group",
+            description="Optional: users in this group are treated as admins. Leave blank to skip group-based admin detection.",
+            placeholder="e.g. admins",
+            default="",
+        ),
+    ]
+
+    oidc_specs = [
+        (
+            TextField,
+            {
+                "key": "OIDC_DISCOVERY_URL",
+                "label": "Discovery URL",
+                "description": "OpenID Connect discovery endpoint URL. Usually ends with /.well-known/openid-configuration.",
+                "placeholder": "https://auth.example.com/.well-known/openid-configuration",
+                "required": True,
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_CLIENT_ID",
+                "label": "Client ID",
+                "description": "OAuth2 client ID from your identity provider.",
+                "placeholder": "shelfmark",
+                "required": True,
+            },
+        ),
+        (
+            PasswordField,
+            {
+                "key": "OIDC_CLIENT_SECRET",
+                "label": "Client Secret",
+                "description": "OAuth2 client secret from your identity provider.",
+                "required": True,
+            },
+        ),
+        (
+            TagListField,
+            {
+                "key": "OIDC_SCOPES",
+                "label": "Scopes",
+                "description": "OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization.",
+                "default": ["openid", "email", "profile"],
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_GROUP_CLAIM",
+                "label": "Group Claim Name",
+                "description": "The name of the claim in the ID token that contains user groups.",
+                "placeholder": "groups",
+                "default": "groups",
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_ADMIN_GROUP",
+                "label": "Admin Group Name",
+                "description": "Users in this group will be given admin access (if enabled below). Leave empty to use database roles only.",
+                "placeholder": "shelfmark-admins",
+                "default": "",
+            },
+        ),
+        (
+            CheckboxField,
+            {
+                "key": "OIDC_USE_ADMIN_GROUP",
+                "label": "Use Admin Group for Authorization",
+                "description": "When enabled, users in the Admin Group are granted admin access. When disabled, admin access is determined solely by database roles.",
+                "default": True,
+            },
+        ),
+        (
+            CheckboxField,
+            {
+                "key": "OIDC_AUTO_PROVISION",
+                "label": "Auto-Provision Users",
+                "description": "Automatically create a user account on first OIDC login. When disabled, users must be pre-created by an admin.",
+                "default": True,
+            },
+        ),
+        (
+            TextField,
+            {
+                "key": "OIDC_BUTTON_LABEL",
+                "label": "Login Button Label",
+                "description": "Custom label for the OIDC sign-in button on the login page.",
+                "placeholder": "Sign in with OIDC",
+                "default": "",
+            },
+        ),
+    ]
+    fields.extend(_auth_ui_field(factory, "oidc", **spec) for factory, spec in oidc_specs)
+    fields.append(
+        ActionButton(
+            key="test_oidc",
+            label="Test Connection",
+            description="Fetch the OIDC discovery document and validate configuration.",
+            style="primary",
+            callback=_test_oidc_connection,
+            show_when=_auth_condition("oidc"),
+        )
+    )
+    return fields
+
+
+register_on_save("security", _on_save_security)

shelfmark/config/security_handlers.py

@@ -0,0 +1,72 @@
+"""Operational handlers for security settings (save/actions)."""
+
+import os
+from typing import Any, Callable
+
+from shelfmark.core.utils import normalize_http_url
+from shelfmark.core.user_db import UserDB
+
+
+_OIDC_LOCKOUT_MESSAGE = "Create a local admin account first (Users tab) before enabling OIDC. This ensures you can still log in with a password if SSO is unavailable."
+
+
+def _has_local_password_admin() -> bool:
+    root = os.environ.get("CONFIG_DIR", "/config")
+    user_db = UserDB(os.path.join(root, "users.db"))
+    user_db.initialize()
+    return any(user.get("password_hash") and user.get("role") == "admin" for user in user_db.list_users())
+
+
+def on_save_security(
+    values: dict[str, Any],
+) -> dict[str, Any]:
+    """Validate security values before persistence."""
+    normalized_values = values.copy()
+
+    discovery_url = normalized_values.get("OIDC_DISCOVERY_URL")
+    if discovery_url is not None:
+        normalized_values["OIDC_DISCOVERY_URL"] = normalize_http_url(
+            str(discovery_url),
+            default_scheme="https",
+        )
+
+    proxy_logout_url = normalized_values.get("PROXY_AUTH_LOGOUT_URL")
+    if proxy_logout_url is not None:
+        normalized_values["PROXY_AUTH_LOGOUT_URL"] = normalize_http_url(
+            str(proxy_logout_url),
+            default_scheme="https",
+            strip_trailing_slash=False,
+        )
+
+    if normalized_values.get("AUTH_METHOD") == "oidc" and not _has_local_password_admin():
+        return {"error": True, "message": _OIDC_LOCKOUT_MESSAGE, "values": normalized_values}
+
+    return {"error": False, "values": normalized_values}
+
+
+def test_oidc_connection(
+    *,
+    load_security_config: Callable[[], dict[str, Any]],
+    logger: Any,
+) -> dict[str, Any]:
+    """Fetch and validate the configured OIDC discovery document."""
+    import requests
+
+    try:
+        discovery_url = load_security_config().get("OIDC_DISCOVERY_URL", "")
+        if not discovery_url:
+            return {"success": False, "message": "Discovery URL is not configured."}
+
+        response = requests.get(discovery_url, timeout=10)
+        response.raise_for_status()
+        document = response.json()
+
+        required_fields = ["issuer", "authorization_endpoint", "token_endpoint"]
+        missing_fields = [field for field in required_fields if field not in document]
+        if missing_fields:
+            return {"success": False, "message": f"Discovery document missing fields: {', '.join(missing_fields)}"}
+
+        return {"success": True, "message": f"Connected to {document['issuer']}"}
+    except Exception as exc:
+        logger.error(f"OIDC connection test failed: {exc}")
+        return {"success": False, "message": f"Connection failed: {str(exc)}"}

shelfmark/config/users_settings.py

@@ -0,0 +1,320 @@
+"""Users settings tab registration.
+
+This registers a 'users' tab in the settings sidebar.
+The actual user management is handled by a custom frontend component
+that talks to /api/admin/users endpoints.
+"""
+
+from shelfmark.core.settings_registry import (
+    CheckboxField,
+    CustomComponentField,
+    HeadingField,
+    MultiSelectField,
+    NumberField,
+    SelectField,
+    TableField,
+    register_on_save,
+    register_settings,
+)
+from shelfmark.core.request_policy import (
+    get_source_content_type_capabilities,
+    parse_policy_mode,
+    validate_policy_rules,
+)
+
+
+_REQUEST_DEFAULT_MODE_OPTIONS = [
+    {
+        "value": "download",
+        "label": "Download",
+        "description": "Everything can be downloaded directly.",
+    },
+    {
+        "value": "request_release",
+        "label": "Request Release",
+        "description": "Users must request a specific release.",
+    },
+    {
+        "value": "request_book",
+        "label": "Request Book",
+        "description": "Users request a book, admin picks the release.",
+    },
+    {
+        "value": "blocked",
+        "label": "Blocked",
+        "description": "No downloads or requests allowed.",
+    },
+]
+
+_REQUEST_MATRIX_MODE_OPTIONS = [
+    option for option in _REQUEST_DEFAULT_MODE_OPTIONS if option["value"] != "request_book"
+]
+
+_SELF_SETTINGS_SECTION_OPTIONS = [
+    {
+        "value": "delivery",
+        "label": "Delivery Preferences",
+        "description": "Show personal delivery output and destination settings.",
+    },
+    {
+        "value": "notifications",
+        "label": "Notifications",
+        "description": "Show personal notification route settings.",
+    },
+]
+_SELF_SETTINGS_SECTION_VALUES = {option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS}
+_SELF_SETTINGS_SECTION_DEFAULTS = [option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS]
+
+_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE = {
+    "builtin": (
+        "Create and manage user accounts directly. Passwords are stored locally and users sign in "
+        "with their username and password."
+    ),
+    "oidc": (
+        "Users sign in through your identity provider. New accounts can be created automatically on "
+        "first login when auto-provisioning is enabled, or you can pre-create users here and they\u2019ll "
+        "be linked by email on first sign-in."
+    ),
+    "proxy": (
+        "Users are authenticated by your reverse proxy. Accounts are automatically created on first "
+        "sign-in. If a local user with a matching username already exists, it will be linked instead."
+    ),
+    "cwa": (
+        "User accounts are synced from your Calibre-Web database. Users are matched by email, and new "
+        "accounts are created here when new CWA users are found."
+    ),
+    "none": "Authentication is disabled. Anyone can access Shelfmark without signing in.",
+    "default": "Authentication is disabled. Anyone can access Shelfmark without signing in.",
+}
+
+
+def _get_request_source_options():
+    """Build request-policy source options from registered release sources."""
+    from shelfmark.release_sources import list_available_sources
+
+    options = []
+    for source in list_available_sources():
+        options.append(
+            {
+                "value": source["name"],
+                "label": source["display_name"],
+            }
+        )
+    return options
+
+
+def _get_request_policy_rule_columns():
+    source_capabilities = get_source_content_type_capabilities()
+    content_type_options = []
+
+    for source_name, supported_types in source_capabilities.items():
+        normalized_types = [t for t in ("ebook", "audiobook") if t in supported_types]
+        for content_type in normalized_types:
+            content_type_options.append(
+                {
+                    "value": content_type,
+                    "label": "Ebook" if content_type == "ebook" else "Audiobook",
+                    "childOf": source_name,
+                }
+            )
+
+    return [
+        {
+            "key": "source",
+            "label": "Source",
+            "type": "select",
+            "options": _get_request_source_options(),
+            "defaultValue": "",
+            "placeholder": "Select source...",
+        },
+        {
+            "key": "content_type",
+            "label": "Content Type",
+            "type": "select",
+            "options": content_type_options,
+            "defaultValue": "",
+            "placeholder": "Select content type...",
+            "filterByField": "source",
+        },
+        {
+            "key": "mode",
+            "label": "Mode",
+            "type": "select",
+            "options": _REQUEST_MATRIX_MODE_OPTIONS,
+            "defaultValue": "",
+            "placeholder": "Select mode...",
+        },
+    ]
+
+
+def _on_save_users(values):
+    """Validate users/request-policy settings before persistence."""
+    if "VISIBLE_SELF_SETTINGS_SECTIONS" in values:
+        raw_sections = values["VISIBLE_SELF_SETTINGS_SECTIONS"]
+        if raw_sections is None:
+            candidate_sections: list[str] = []
+        elif isinstance(raw_sections, str):
+            candidate_sections = [s.strip() for s in raw_sections.split(",") if s.strip()]
+        elif isinstance(raw_sections, (list, tuple, set)):
+            candidate_sections = [str(section).strip() for section in raw_sections if str(section).strip()]
+        else:
+            return {
+                "error": True,
+                "message": "VISIBLE_SELF_SETTINGS_SECTIONS must be a list of section identifiers",
+                "values": values,
+            }
+
+        normalized_sections: list[str] = []
+        for section in candidate_sections:
+            if section not in _SELF_SETTINGS_SECTION_VALUES:
+                allowed = ", ".join(sorted(_SELF_SETTINGS_SECTION_VALUES))
+                return {
+                    "error": True,
+                    "message": (
+                        "VISIBLE_SELF_SETTINGS_SECTIONS contains an unsupported section "
+                        f"'{section}'. Supported values: {allowed}"
+                    ),
+                    "values": values,
+                }
+            if section not in normalized_sections:
+                normalized_sections.append(section)
+
+        values["VISIBLE_SELF_SETTINGS_SECTIONS"] = normalized_sections
+
+    if "REQUEST_POLICY_DEFAULT_EBOOK" in values:
+        if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_EBOOK"]) is None:
+            return {
+                "error": True,
+                "message": "REQUEST_POLICY_DEFAULT_EBOOK must be a valid policy mode",
+                "values": values,
+            }
+
+    if "REQUEST_POLICY_DEFAULT_AUDIOBOOK" in values:
+        if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_AUDIOBOOK"]) is None:
+            return {
+                "error": True,
+                "message": "REQUEST_POLICY_DEFAULT_AUDIOBOOK must be a valid policy mode",
+                "values": values,
+            }
+
+    if "REQUEST_POLICY_RULES" in values:
+        normalized_rules, errors = validate_policy_rules(values["REQUEST_POLICY_RULES"])
+        if errors:
+            return {
+                "error": True,
+                "message": "; ".join(errors),
+                "values": values,
+            }
+        values["REQUEST_POLICY_RULES"] = normalized_rules
+
+    return {"error": False, "values": values}
+
+
+register_on_save("users", _on_save_users)
+
+
+@register_settings("users", "Users & Requests", icon="users", order=6)
+def users_settings():
+    """User management tab - rendered as a custom component on the frontend."""
+    return [
+        HeadingField(
+            key="users_heading",
+            title="Users",
+            description=_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE["default"],
+            description_by_auth_mode=_USERS_HEADING_DESCRIPTION_BY_AUTH_MODE,
+        ),
+        CustomComponentField(
+            key="users_management",
+            component="users_management",
+        ),
+        MultiSelectField(
+            key="VISIBLE_SELF_SETTINGS_SECTIONS",
+            label="Visible Self-Settings Sections",
+            description=(
+                "Choose which personal settings sections are shown in My Account for non-admin users."
+            ),
+            options=_SELF_SETTINGS_SECTION_OPTIONS,
+            default=_SELF_SETTINGS_SECTION_DEFAULTS,
+            variant="dropdown",
+            env_supported=False,
+        ),
+        HeadingField(
+            key="requests_heading",
+            title="Requests",
+            description=(
+                "Choose what users can download directly and what needs approval first."
+            ),
+        ),
+        CheckboxField(
+            key="REQUESTS_ENABLED",
+            label="Enable Requests",
+            description=(
+                "Turn this off to let everyone download directly without needing approval."
+            ),
+            default=False,
+            user_overridable=True,
+        ),
+        CustomComponentField(
+            key="request_policy_editor",
+            component="request_policy_grid",
+            label="Request Rules",
+            description=(
+                "Fine-tune access per source. Source rules can only be the same or more restrictive than the default above."
+            ),
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+            wrap_in_field_wrapper=True,
+            value_fields=[
+                SelectField(
+                    key="REQUEST_POLICY_DEFAULT_EBOOK",
+                    label="Default Ebook Mode",
+                    description=(
+                        "Sets the baseline for all ebook sources."
+                    ),
+                    options=_REQUEST_DEFAULT_MODE_OPTIONS,
+                    default="download",
+                    user_overridable=True,
+                ),
+                SelectField(
+                    key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
+                    label="Default Audiobook Mode",
+                    description=(
+                        "Sets the baseline for all audiobook sources."
+                    ),
+                    options=_REQUEST_DEFAULT_MODE_OPTIONS,
+                    default="download",
+                    user_overridable=True,
+                ),
+                TableField(
+                    key="REQUEST_POLICY_RULES",
+                    label="Request Rules",
+                    description=(
+                        "Fine-tune access per source. Source rules can only be the same or more restrictive than the default above."
+                    ),
+                    columns=_get_request_policy_rule_columns,
+                    default=[],
+                    add_label="Add Rule",
+                    empty_message="No request policy rules configured.",
+                    env_supported=False,
+                    user_overridable=True,
+                ),
+            ],
+        ),
+        NumberField(
+            key="MAX_PENDING_REQUESTS_PER_USER",
+            label="Max pending requests per user",
+            description="How many open requests a user can have at a time.",
+            default=20,
+            min_value=1,
+            max_value=1000,
+            user_overridable=True,
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+        ),
+        CheckboxField(
+            key="REQUESTS_ALLOW_NOTES",
+            label="Allow notes on requests",
+            description="Let users add a note when they submit a request.",
+            default=True,
+            user_overridable=True,
+            show_when={"field": "REQUESTS_ENABLED", "value": True},
+        ),
+    ]

shelfmark/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core module - shared models, queue, and utilities."""
+
+from shelfmark.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus
+from shelfmark.core.queue import BookQueue, book_queue
+from shelfmark.core.logger import setup_logger

shelfmark/core/activity_routes.py

@@ -0,0 +1,486 @@
+"""Activity API routes (snapshot, dismiss, history)."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from flask import Flask, jsonify, request, session
+
+from shelfmark.core.activity_service import ActivityService
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+
+
+def _require_authenticated(resolve_auth_mode: Callable[[], str]):
+    auth_mode = resolve_auth_mode()
+    if auth_mode == "none":
+        return None
+    if "user_id" not in session:
+        return jsonify({"error": "Unauthorized"}), 401
+    return None
+
+
+def _resolve_db_user_id(require_in_auth_mode: bool = True):
+    raw_db_user_id = session.get("db_user_id")
+    if raw_db_user_id is None:
+        if not require_in_auth_mode:
+            return None, None
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+    try:
+        return int(raw_db_user_id), None
+    except (TypeError, ValueError):
+        return None, (
+            jsonify(
+                {
+                    "error": "User identity unavailable for activity workflow",
+                    "code": "user_identity_unavailable",
+                }
+            ),
+            403,
+        )
+
+
+def _emit_activity_event(ws_manager: Any | None, *, room: str, payload: dict[str, Any]) -> None:
+    if ws_manager is None:
+        return
+    try:
+        socketio = getattr(ws_manager, "socketio", None)
+        is_enabled = getattr(ws_manager, "is_enabled", None)
+        if socketio is None or not callable(is_enabled) or not is_enabled():
+            return
+        socketio.emit("activity_update", payload, to=room)
+    except Exception as exc:
+        logger.warning("Failed to emit activity_update event: %s", exc)
+
+
+def _list_visible_requests(user_db: UserDB, *, is_admin: bool, db_user_id: int | None) -> list[dict[str, Any]]:
+    if is_admin:
+        request_rows = user_db.list_requests()
+        user_cache: dict[int, str] = {}
+        for row in request_rows:
+            requester_id = row["user_id"]
+            if requester_id not in user_cache:
+                requester = user_db.get_user(user_id=requester_id)
+                user_cache[requester_id] = requester.get("username", "") if requester else ""
+            row["username"] = user_cache[requester_id]
+        return request_rows
+
+    if db_user_id is None:
+        return []
+    return user_db.list_requests(user_id=db_user_id)
+
+
+def _parse_download_item_key(item_key: str) -> str | None:
+    if not isinstance(item_key, str) or not item_key.startswith("download:"):
+        return None
+    task_id = item_key.split(":", 1)[1].strip()
+    return task_id or None
+
+
+def _parse_request_item_key(item_key: str) -> int | None:
+    if not isinstance(item_key, str) or not item_key.startswith("request:"):
+        return None
+    raw_id = item_key.split(":", 1)[1].strip()
+    try:
+        parsed = int(raw_id)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _task_id_from_download_item_key(item_key: str) -> str | None:
+    task_id = _parse_download_item_key(item_key)
+    if task_id is None:
+        return None
+    return task_id
+
+
+def _merge_terminal_snapshot_backfill(
+    *,
+    status: dict[str, dict[str, Any]],
+    terminal_rows: list[dict[str, Any]],
+) -> None:
+    existing_task_ids: set[str] = set()
+    for bucket_key in ("queued", "resolving", "locating", "downloading", "complete", "error", "cancelled"):
+        bucket = status.get(bucket_key)
+        if not isinstance(bucket, dict):
+            continue
+        existing_task_ids.update(str(task_id) for task_id in bucket.keys())
+
+    for row in terminal_rows:
+        item_key = row.get("item_key")
+        if not isinstance(item_key, str):
+            continue
+        task_id = _task_id_from_download_item_key(item_key)
+        if not task_id or task_id in existing_task_ids:
+            continue
+
+        final_status = row.get("final_status")
+        if final_status not in {"complete", "error", "cancelled"}:
+            continue
+
+        snapshot = row.get("snapshot")
+        if not isinstance(snapshot, dict):
+            continue
+        raw_download = snapshot.get("download")
+        if not isinstance(raw_download, dict):
+            continue
+
+        download_payload = dict(raw_download)
+        if not isinstance(download_payload.get("id"), str):
+            download_payload["id"] = task_id
+
+        if final_status not in status or not isinstance(status.get(final_status), dict):
+            status[final_status] = {}
+        status[final_status][task_id] = download_payload
+        existing_task_ids.add(task_id)
+
+
+def _collect_active_download_item_keys(status: dict[str, dict[str, Any]]) -> set[str]:
+    active_keys: set[str] = set()
+    for bucket_key in ("queued", "resolving", "locating", "downloading"):
+        bucket = status.get(bucket_key)
+        if not isinstance(bucket, dict):
+            continue
+        for task_id in bucket.keys():
+            normalized_task_id = str(task_id).strip()
+            if not normalized_task_id:
+                continue
+            active_keys.add(f"download:{normalized_task_id}")
+    return active_keys
+
+
+def _extract_request_source_id(row: dict[str, Any]) -> str | None:
+    release_data = row.get("release_data")
+    if not isinstance(release_data, dict):
+        return None
+    source_id = release_data.get("source_id")
+    if not isinstance(source_id, str):
+        return None
+    normalized = source_id.strip()
+    return normalized or None
+
+
+def _request_terminal_status(row: dict[str, Any]) -> str | None:
+    request_status = row.get("status")
+    if request_status == "pending":
+        return None
+    if request_status == "rejected":
+        return "rejected"
+    if request_status == "cancelled":
+        return "cancelled"
+    if request_status != "fulfilled":
+        return None
+
+    delivery_state = str(row.get("delivery_state") or "").strip().lower()
+    if delivery_state in {"error", "cancelled"}:
+        return delivery_state
+    return "complete"
+
+
+def _minimal_request_snapshot(request_row: dict[str, Any], request_id: int) -> dict[str, Any]:
+    book_data = request_row.get("book_data")
+    release_data = request_row.get("release_data")
+    if not isinstance(book_data, dict):
+        book_data = {}
+    if not isinstance(release_data, dict):
+        release_data = {}
+
+    minimal_request = {
+        "id": request_id,
+        "user_id": request_row.get("user_id"),
+        "status": request_row.get("status"),
+        "request_level": request_row.get("request_level"),
+        "delivery_state": request_row.get("delivery_state"),
+        "book_data": book_data,
+        "release_data": release_data,
+        "note": request_row.get("note"),
+        "admin_note": request_row.get("admin_note"),
+        "created_at": request_row.get("created_at"),
+        "updated_at": request_row.get("updated_at"),
+    }
+    username = request_row.get("username")
+    if isinstance(username, str):
+        minimal_request["username"] = username
+    return {"kind": "request", "request": minimal_request}
+
+
+def _get_existing_activity_log_id_for_item(
+    *,
+    activity_service: ActivityService,
+    user_db: UserDB,
+    item_type: str,
+    item_key: str,
+) -> int | None:
+    if item_type not in {"request", "download"}:
+        return None
+    if not isinstance(item_key, str) or not item_key.strip():
+        return None
+
+    existing_log_id = activity_service.get_latest_activity_log_id(
+        item_type=item_type,
+        item_key=item_key,
+    )
+    if existing_log_id is not None or item_type != "request":
+        return existing_log_id
+
+    request_id = _parse_request_item_key(item_key)
+    if request_id is None:
+        return None
+    row = user_db.get_request(request_id)
+    if row is None:
+        return None
+
+    final_status = _request_terminal_status(row)
+    if final_status is None:
+        return None
+
+    source_id = _extract_request_source_id(row)
+    payload = activity_service.record_terminal_snapshot(
+        user_id=row.get("user_id"),
+        item_type="request",
+        item_key=item_key,
+        origin="request",
+        final_status=final_status,
+        snapshot=_minimal_request_snapshot(row, request_id),
+        request_id=request_id,
+        source_id=source_id,
+    )
+    return int(payload["id"])
+
+
+def register_activity_routes(
+    app: Flask,
+    user_db: UserDB,
+    *,
+    activity_service: ActivityService,
+    resolve_auth_mode: Callable[[], str],
+    resolve_status_scope: Callable[[], tuple[bool, int | None, bool]],
+    queue_status: Callable[..., dict[str, dict[str, Any]]],
+    sync_request_delivery_states: Callable[..., list[dict[str, Any]]],
+    emit_request_updates: Callable[[list[dict[str, Any]]], None],
+    ws_manager: Any | None = None,
+) -> None:
+    """Register activity routes."""
+
+    @app.route("/api/activity/snapshot", methods=["GET"])
+    def api_activity_snapshot():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        is_admin, db_user_id, can_access_status = resolve_status_scope()
+        if not can_access_status:
+            return (
+                jsonify(
+                    {
+                        "error": "User identity unavailable for activity workflow",
+                        "code": "user_identity_unavailable",
+                    }
+                ),
+                403,
+            )
+
+        viewer_db_user_id, _ = _resolve_db_user_id(require_in_auth_mode=False)
+        scoped_user_id = None if is_admin else db_user_id
+        status = queue_status(user_id=scoped_user_id)
+        updated_requests = sync_request_delivery_states(
+            user_db,
+            queue_status=status,
+            user_id=scoped_user_id,
+        )
+        emit_request_updates(updated_requests)
+        request_rows = _list_visible_requests(user_db, is_admin=is_admin, db_user_id=db_user_id)
+
+        if not is_admin and db_user_id is not None:
+            try:
+                terminal_rows = activity_service.get_undismissed_terminal_downloads(db_user_id, limit=200)
+                _merge_terminal_snapshot_backfill(status=status, terminal_rows=terminal_rows)
+            except Exception as exc:
+                logger.warning("Failed to merge terminal snapshot backfill rows: %s", exc)
+
+        if viewer_db_user_id is not None:
+            active_download_keys = _collect_active_download_item_keys(status)
+            if active_download_keys:
+                try:
+                    activity_service.clear_dismissals_for_item_keys(
+                        user_id=viewer_db_user_id,
+                        item_type="download",
+                        item_keys=active_download_keys,
+                    )
+                except Exception as exc:
+                    logger.warning("Failed to clear stale download dismissals for active tasks: %s", exc)
+
+        dismissed: list[dict[str, str]] = []
+        # Admins can view unscoped queue status, but dismissals remain per-viewer.
+        if viewer_db_user_id is not None:
+            dismissed = activity_service.get_dismissal_set(viewer_db_user_id)
+
+        return jsonify(
+            {
+                "status": status,
+                "requests": request_rows,
+                "dismissed": dismissed,
+            }
+        )
+
+    @app.route("/api/activity/dismiss", methods=["POST"])
+    def api_activity_dismiss():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        data = request.get_json(silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": "Invalid payload"}), 400
+
+        activity_log_id = data.get("activity_log_id")
+        if activity_log_id is None:
+            try:
+                activity_log_id = _get_existing_activity_log_id_for_item(
+                    activity_service=activity_service,
+                    user_db=user_db,
+                    item_type=data.get("item_type"),
+                    item_key=data.get("item_key"),
+                )
+            except Exception as exc:
+                logger.warning("Failed to resolve activity snapshot id for dismiss payload: %s", exc)
+                activity_log_id = None
+
+        try:
+            dismissal = activity_service.dismiss_item(
+                user_id=db_user_id,
+                item_type=data.get("item_type"),
+                item_key=data.get("item_key"),
+                activity_log_id=activity_log_id,
+            )
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+
+        _emit_activity_event(
+            ws_manager,
+            room=f"user_{db_user_id}",
+            payload={
+                "kind": "dismiss",
+                "user_id": db_user_id,
+                "item_type": dismissal["item_type"],
+                "item_key": dismissal["item_key"],
+            },
+        )
+
+        return jsonify({"status": "dismissed", "item": dismissal})
+
+    @app.route("/api/activity/dismiss-many", methods=["POST"])
+    def api_activity_dismiss_many():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        data = request.get_json(silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": "Invalid payload"}), 400
+        items = data.get("items")
+        if not isinstance(items, list):
+            return jsonify({"error": "items must be an array"}), 400
+
+        normalized_items: list[dict[str, Any]] = []
+        for item in items:
+            if not isinstance(item, dict):
+                return jsonify({"error": "items must contain objects"}), 400
+
+            activity_log_id = item.get("activity_log_id")
+            if activity_log_id is None:
+                try:
+                    activity_log_id = _get_existing_activity_log_id_for_item(
+                        activity_service=activity_service,
+                        user_db=user_db,
+                        item_type=item.get("item_type"),
+                        item_key=item.get("item_key"),
+                    )
+                except Exception as exc:
+                    logger.warning("Failed to resolve activity snapshot id for dismiss-many item: %s", exc)
+                    activity_log_id = None
+
+            normalized_payload = {
+                "item_type": item.get("item_type"),
+                "item_key": item.get("item_key"),
+            }
+            if activity_log_id is not None:
+                normalized_payload["activity_log_id"] = activity_log_id
+            normalized_items.append(normalized_payload)
+
+        try:
+            dismissed_count = activity_service.dismiss_many(user_id=db_user_id, items=normalized_items)
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+
+        _emit_activity_event(
+            ws_manager,
+            room=f"user_{db_user_id}",
+            payload={
+                "kind": "dismiss_many",
+                "user_id": db_user_id,
+                "count": dismissed_count,
+            },
+        )
+
+        return jsonify({"status": "dismissed", "count": dismissed_count})
+
+    @app.route("/api/activity/history", methods=["GET"])
+    def api_activity_history():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        limit = request.args.get("limit", type=int, default=50) or 50
+        offset = request.args.get("offset", type=int, default=0) or 0
+
+        try:
+            history = activity_service.get_history(db_user_id, limit=limit, offset=offset)
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+        return jsonify(history)
+
+    @app.route("/api/activity/history", methods=["DELETE"])
+    def api_activity_history_clear():
+        auth_gate = _require_authenticated(resolve_auth_mode)
+        if auth_gate is not None:
+            return auth_gate
+
+        db_user_id, db_gate = _resolve_db_user_id()
+        if db_gate is not None or db_user_id is None:
+            return db_gate
+
+        deleted_count = activity_service.clear_history(db_user_id)
+        _emit_activity_event(
+            ws_manager,
+            room=f"user_{db_user_id}",
+            payload={
+                "kind": "history_cleared",
+                "user_id": db_user_id,
+                "count": deleted_count,
+            },
+        )
+        return jsonify({"status": "cleared", "deleted_count": deleted_count})

shelfmark/core/activity_service.py

@@ -0,0 +1,618 @@
+"""Persistence helpers for Activity dismissals and terminal snapshots."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import sqlite3
+from typing import Any, Iterable
+
+
+VALID_ITEM_TYPES = frozenset({"download", "request"})
+VALID_ORIGINS = frozenset({"direct", "request", "requested"})
+VALID_FINAL_STATUSES = frozenset({"complete", "error", "cancelled", "rejected"})
+
+
+def _now_timestamp() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _normalize_item_type(item_type: Any) -> str:
+    if not isinstance(item_type, str):
+        raise ValueError("item_type must be a string")
+    normalized = item_type.strip().lower()
+    if normalized not in VALID_ITEM_TYPES:
+        raise ValueError("item_type must be one of: download, request")
+    return normalized
+
+
+def _normalize_item_key(item_key: Any) -> str:
+    if not isinstance(item_key, str):
+        raise ValueError("item_key must be a string")
+    normalized = item_key.strip()
+    if not normalized:
+        raise ValueError("item_key must not be empty")
+    return normalized
+
+
+def _normalize_origin(origin: Any) -> str:
+    if not isinstance(origin, str):
+        raise ValueError("origin must be a string")
+    normalized = origin.strip().lower()
+    if normalized not in VALID_ORIGINS:
+        raise ValueError("origin must be one of: direct, request, requested")
+    return normalized
+
+
+def _normalize_final_status(final_status: Any) -> str:
+    if not isinstance(final_status, str):
+        raise ValueError("final_status must be a string")
+    normalized = final_status.strip().lower()
+    if normalized not in VALID_FINAL_STATUSES:
+        raise ValueError("final_status must be one of: complete, error, cancelled, rejected")
+    return normalized
+
+
+def build_item_key(item_type: str, raw_id: Any) -> str:
+    """Build a stable item key used by dismiss/history APIs."""
+    normalized_type = _normalize_item_type(item_type)
+    if normalized_type == "request":
+        try:
+            request_id = int(raw_id)
+        except (TypeError, ValueError) as exc:
+            raise ValueError("request item IDs must be integers") from exc
+        if request_id < 1:
+            raise ValueError("request item IDs must be positive integers")
+        return f"request:{request_id}"
+
+    if not isinstance(raw_id, str):
+        raise ValueError("download item IDs must be strings")
+    task_id = raw_id.strip()
+    if not task_id:
+        raise ValueError("download item IDs must not be empty")
+    return f"download:{task_id}"
+
+
+def build_request_item_key(request_id: int) -> str:
+    """Build a request item key."""
+    return build_item_key("request", request_id)
+
+
+def build_download_item_key(task_id: str) -> str:
+    """Build a download item key."""
+    return build_item_key("download", task_id)
+
+
+def _parse_request_id_from_item_key(item_key: Any) -> int | None:
+    if not isinstance(item_key, str) or not item_key.startswith("request:"):
+        return None
+    raw_value = item_key.split(":", 1)[1].strip()
+    try:
+        parsed = int(raw_value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _request_final_status(request_status: Any, delivery_state: Any) -> str | None:
+    status = str(request_status or "").strip().lower()
+    if status == "pending":
+        return None
+    if status == "rejected":
+        return "rejected"
+    if status == "cancelled":
+        return "cancelled"
+    if status != "fulfilled":
+        return None
+
+    delivery = str(delivery_state or "").strip().lower()
+    if delivery in {"error", "cancelled"}:
+        return delivery
+    return "complete"
+
+
+class ActivityService:
+    """Service for per-user activity dismissals and terminal history snapshots."""
+
+    def __init__(self, db_path: str):
+        self._db_path = db_path
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA foreign_keys = ON")
+        return conn
+
+    @staticmethod
+    def _coerce_positive_int(value: Any, field: str) -> int:
+        try:
+            parsed = int(value)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"{field} must be an integer") from exc
+        if parsed < 1:
+            raise ValueError(f"{field} must be a positive integer")
+        return parsed
+
+    @staticmethod
+    def _row_to_dict(row: sqlite3.Row | None) -> dict[str, Any] | None:
+        return dict(row) if row is not None else None
+
+    @staticmethod
+    def _parse_json_column(value: Any) -> Any:
+        if not isinstance(value, str):
+            return None
+        try:
+            return json.loads(value)
+        except (ValueError, TypeError):
+            return None
+
+    def _build_legacy_request_snapshot(
+        self,
+        conn: sqlite3.Connection,
+        request_id: int,
+    ) -> tuple[dict[str, Any] | None, str | None]:
+        request_row = conn.execute(
+            """
+            SELECT
+                id,
+                user_id,
+                status,
+                delivery_state,
+                request_level,
+                book_data,
+                release_data,
+                note,
+                admin_note,
+                created_at,
+                reviewed_at
+            FROM download_requests
+            WHERE id = ?
+            """,
+            (request_id,),
+        ).fetchone()
+        if request_row is None:
+            return None, None
+
+        row_dict = dict(request_row)
+        book_data = self._parse_json_column(row_dict.get("book_data"))
+        release_data = self._parse_json_column(row_dict.get("release_data"))
+        if not isinstance(book_data, dict):
+            book_data = {}
+        if not isinstance(release_data, dict):
+            release_data = {}
+
+        snapshot = {
+            "kind": "request",
+            "request": {
+                "id": int(row_dict["id"]),
+                "user_id": row_dict.get("user_id"),
+                "status": row_dict.get("status"),
+                "delivery_state": row_dict.get("delivery_state"),
+                "request_level": row_dict.get("request_level"),
+                "book_data": book_data,
+                "release_data": release_data,
+                "note": row_dict.get("note"),
+                "admin_note": row_dict.get("admin_note"),
+                "created_at": row_dict.get("created_at"),
+                "updated_at": row_dict.get("reviewed_at") or row_dict.get("created_at"),
+            },
+        }
+        final_status = _request_final_status(row_dict.get("status"), row_dict.get("delivery_state"))
+        return snapshot, final_status
+
+    def record_terminal_snapshot(
+        self,
+        *,
+        user_id: int | None,
+        item_type: str,
+        item_key: str,
+        origin: str,
+        final_status: str,
+        snapshot: dict[str, Any],
+        request_id: int | None = None,
+        source_id: str | None = None,
+        terminal_at: str | None = None,
+    ) -> dict[str, Any]:
+        """Record a durable terminal-state snapshot for an activity item."""
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_item_key = _normalize_item_key(item_key)
+        normalized_origin = _normalize_origin(origin)
+        normalized_final_status = _normalize_final_status(final_status)
+        if not isinstance(snapshot, dict):
+            raise ValueError("snapshot must be an object")
+
+        if user_id is not None:
+            user_id = self._coerce_positive_int(user_id, "user_id")
+        if request_id is not None:
+            request_id = self._coerce_positive_int(request_id, "request_id")
+        if source_id is not None and not isinstance(source_id, str):
+            raise ValueError("source_id must be a string when provided")
+        if source_id is not None:
+            source_id = source_id.strip() or None
+
+        effective_terminal_at = terminal_at if isinstance(terminal_at, str) and terminal_at.strip() else _now_timestamp()
+        serialized_snapshot = json.dumps(snapshot, separators=(",", ":"), ensure_ascii=False)
+
+        conn = self._connect()
+        try:
+            cursor = conn.execute(
+                """
+                INSERT INTO activity_log (
+                    user_id,
+                    item_type,
+                    item_key,
+                    request_id,
+                    source_id,
+                    origin,
+                    final_status,
+                    snapshot_json,
+                    terminal_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    user_id,
+                    normalized_item_type,
+                    normalized_item_key,
+                    request_id,
+                    source_id,
+                    normalized_origin,
+                    normalized_final_status,
+                    serialized_snapshot,
+                    effective_terminal_at,
+                ),
+            )
+            snapshot_id = int(cursor.lastrowid)
+            conn.commit()
+            row = conn.execute(
+                "SELECT * FROM activity_log WHERE id = ?",
+                (snapshot_id,),
+            ).fetchone()
+            payload = self._row_to_dict(row)
+            if payload is None:
+                raise ValueError("Failed to read back recorded activity snapshot")
+            return payload
+        finally:
+            conn.close()
+
+    def get_latest_activity_log_id(self, *, item_type: str, item_key: str) -> int | None:
+        """Get the newest snapshot ID for an item key."""
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_item_key = _normalize_item_key(item_key)
+        conn = self._connect()
+        try:
+            row = conn.execute(
+                """
+                SELECT id
+                FROM activity_log
+                WHERE item_type = ? AND item_key = ?
+                ORDER BY terminal_at DESC, id DESC
+                LIMIT 1
+                """,
+                (normalized_item_type, normalized_item_key),
+            ).fetchone()
+            if row is None:
+                return None
+            return int(row["id"])
+        finally:
+            conn.close()
+
+    def dismiss_item(
+        self,
+        *,
+        user_id: int,
+        item_type: str,
+        item_key: str,
+        activity_log_id: int | None = None,
+    ) -> dict[str, Any]:
+        """Dismiss an item for a specific user (upsert)."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_item_key = _normalize_item_key(item_key)
+        normalized_log_id = (
+            self._coerce_positive_int(activity_log_id, "activity_log_id")
+            if activity_log_id is not None
+            else self.get_latest_activity_log_id(
+                item_type=normalized_item_type,
+                item_key=normalized_item_key,
+            )
+        )
+
+        conn = self._connect()
+        try:
+            conn.execute(
+                """
+                INSERT INTO activity_dismissals (
+                    user_id,
+                    item_type,
+                    item_key,
+                    activity_log_id,
+                    dismissed_at
+                )
+                VALUES (?, ?, ?, ?, ?)
+                ON CONFLICT(user_id, item_type, item_key)
+                DO UPDATE SET
+                    activity_log_id = excluded.activity_log_id,
+                    dismissed_at = excluded.dismissed_at
+                """,
+                (
+                    normalized_user_id,
+                    normalized_item_type,
+                    normalized_item_key,
+                    normalized_log_id,
+                    _now_timestamp(),
+                ),
+            )
+            conn.commit()
+            row = conn.execute(
+                """
+                SELECT *
+                FROM activity_dismissals
+                WHERE user_id = ? AND item_type = ? AND item_key = ?
+                """,
+                (normalized_user_id, normalized_item_type, normalized_item_key),
+            ).fetchone()
+            payload = self._row_to_dict(row)
+            if payload is None:
+                raise ValueError("Failed to read back dismissal row")
+            return payload
+        finally:
+            conn.close()
+
+    def dismiss_many(self, *, user_id: int, items: Iterable[dict[str, Any]]) -> int:
+        """Dismiss many items for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_items: list[tuple[str, str, int | None]] = []
+        for item in items:
+            if not isinstance(item, dict):
+                raise ValueError("items must contain objects")
+            normalized_item_type = _normalize_item_type(item.get("item_type"))
+            normalized_item_key = _normalize_item_key(item.get("item_key"))
+            raw_log_id = item.get("activity_log_id")
+            normalized_log_id = (
+                self._coerce_positive_int(raw_log_id, "activity_log_id")
+                if raw_log_id is not None
+                else self.get_latest_activity_log_id(
+                    item_type=normalized_item_type,
+                    item_key=normalized_item_key,
+                )
+            )
+            normalized_items.append((normalized_item_type, normalized_item_key, normalized_log_id))
+
+        if not normalized_items:
+            return 0
+
+        conn = self._connect()
+        try:
+            timestamp = _now_timestamp()
+            for item_type, item_key, activity_log_id in normalized_items:
+                conn.execute(
+                    """
+                    INSERT INTO activity_dismissals (
+                        user_id,
+                        item_type,
+                        item_key,
+                        activity_log_id,
+                        dismissed_at
+                    )
+                    VALUES (?, ?, ?, ?, ?)
+                    ON CONFLICT(user_id, item_type, item_key)
+                    DO UPDATE SET
+                        activity_log_id = excluded.activity_log_id,
+                        dismissed_at = excluded.dismissed_at
+                    """,
+                    (
+                        normalized_user_id,
+                        item_type,
+                        item_key,
+                        activity_log_id,
+                        timestamp,
+                    ),
+                )
+            conn.commit()
+            return len(normalized_items)
+        finally:
+            conn.close()
+
+    def get_dismissal_set(self, user_id: int) -> list[dict[str, str]]:
+        """Return dismissed item keys for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT item_type, item_key
+                FROM activity_dismissals
+                WHERE user_id = ?
+                ORDER BY dismissed_at DESC, id DESC
+                """,
+                (normalized_user_id,),
+            ).fetchall()
+            return [
+                {
+                    "item_type": str(row["item_type"]),
+                    "item_key": str(row["item_key"]),
+                }
+                for row in rows
+            ]
+        finally:
+            conn.close()
+
+    def clear_dismissals_for_item_keys(
+        self,
+        *,
+        user_id: int,
+        item_type: str,
+        item_keys: Iterable[str],
+    ) -> int:
+        """Clear dismissals for one user + item type + item keys."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_item_type = _normalize_item_type(item_type)
+        normalized_keys = {
+            _normalize_item_key(item_key)
+            for item_key in item_keys
+            if isinstance(item_key, str) and item_key.strip()
+        }
+        if not normalized_keys:
+            return 0
+
+        conn = self._connect()
+        try:
+            cursor = conn.executemany(
+                """
+                DELETE FROM activity_dismissals
+                WHERE user_id = ? AND item_type = ? AND item_key = ?
+                """,
+                (
+                    (normalized_user_id, normalized_item_type, item_key)
+                    for item_key in normalized_keys
+                ),
+            )
+            conn.commit()
+            return int(cursor.rowcount or 0)
+        finally:
+            conn.close()
+
+    def get_history(self, user_id: int, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
+        """Return paged dismissal history for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_limit = max(1, min(int(limit), 200))
+        normalized_offset = max(0, int(offset))
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT
+                    d.id,
+                    d.user_id,
+                    d.item_type,
+                    d.item_key,
+                    d.activity_log_id,
+                    d.dismissed_at,
+                    l.snapshot_json,
+                    l.origin,
+                    l.final_status,
+                    l.terminal_at,
+                    l.request_id,
+                    l.source_id
+                FROM activity_dismissals d
+                LEFT JOIN activity_log l ON l.id = d.activity_log_id
+                WHERE d.user_id = ?
+                ORDER BY d.dismissed_at DESC, d.id DESC
+                LIMIT ? OFFSET ?
+                """,
+                (normalized_user_id, normalized_limit, normalized_offset),
+            ).fetchall()
+
+            payload: list[dict[str, Any]] = []
+            for row in rows:
+                row_dict = dict(row)
+                raw_snapshot_json = row_dict.pop("snapshot_json", None)
+                snapshot_payload = None
+                if isinstance(raw_snapshot_json, str):
+                    try:
+                        snapshot_payload = json.loads(raw_snapshot_json)
+                    except (ValueError, TypeError):
+                        snapshot_payload = None
+
+                if snapshot_payload is None and row_dict.get("item_type") == "request":
+                    request_id = row_dict.get("request_id")
+                    if request_id is None:
+                        request_id = _parse_request_id_from_item_key(row_dict.get("item_key"))
+                    try:
+                        normalized_request_id = int(request_id) if request_id is not None else None
+                    except (TypeError, ValueError):
+                        normalized_request_id = None
+
+                    if normalized_request_id and normalized_request_id > 0:
+                        fallback_snapshot, fallback_final_status = self._build_legacy_request_snapshot(
+                            conn,
+                            normalized_request_id,
+                        )
+                        if fallback_snapshot is not None:
+                            snapshot_payload = fallback_snapshot
+                            if not row_dict.get("origin"):
+                                row_dict["origin"] = "request"
+                            if not row_dict.get("final_status") and fallback_final_status is not None:
+                                row_dict["final_status"] = fallback_final_status
+
+                row_dict["snapshot"] = snapshot_payload
+                payload.append(row_dict)
+            return payload
+        finally:
+            conn.close()
+
+    def get_undismissed_terminal_downloads(self, user_id: int, *, limit: int = 200) -> list[dict[str, Any]]:
+        """Return latest undismissed terminal download snapshots for one user."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        normalized_limit = max(1, min(int(limit), 500))
+
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """
+                SELECT
+                    l.id,
+                    l.user_id,
+                    l.item_type,
+                    l.item_key,
+                    l.request_id,
+                    l.source_id,
+                    l.origin,
+                    l.final_status,
+                    l.snapshot_json,
+                    l.terminal_at
+                FROM activity_log l
+                LEFT JOIN activity_dismissals d
+                    ON d.user_id = ?
+                    AND d.item_type = l.item_type
+                    AND d.item_key = l.item_key
+                WHERE l.user_id = ?
+                    AND l.item_type = 'download'
+                    AND l.final_status IN ('complete', 'error', 'cancelled')
+                    AND d.id IS NULL
+                ORDER BY l.terminal_at DESC, l.id DESC
+                LIMIT ?
+                """,
+                (normalized_user_id, normalized_user_id, normalized_limit * 2),
+            ).fetchall()
+
+            payload: list[dict[str, Any]] = []
+            seen_item_keys: set[str] = set()
+            for row in rows:
+                row_dict = dict(row)
+                item_key = str(row_dict.get("item_key") or "")
+                if not item_key or item_key in seen_item_keys:
+                    continue
+                seen_item_keys.add(item_key)
+
+                raw_snapshot_json = row_dict.pop("snapshot_json", None)
+                snapshot_payload = None
+                if isinstance(raw_snapshot_json, str):
+                    try:
+                        snapshot_payload = json.loads(raw_snapshot_json)
+                    except (ValueError, TypeError):
+                        snapshot_payload = None
+                row_dict["snapshot"] = snapshot_payload
+                payload.append(row_dict)
+                if len(payload) >= normalized_limit:
+                    break
+
+            return payload
+        finally:
+            conn.close()
+
+    def clear_history(self, user_id: int) -> int:
+        """Delete all dismissals for a user and return deleted row count."""
+        normalized_user_id = self._coerce_positive_int(user_id, "user_id")
+        conn = self._connect()
+        try:
+            cursor = conn.execute(
+                "DELETE FROM activity_dismissals WHERE user_id = ?",
+                (normalized_user_id,),
+            )
+            conn.commit()
+            return int(cursor.rowcount or 0)
+        finally:
+            conn.close()

shelfmark/core/admin_routes.py

@@ -0,0 +1,439 @@
+"""Admin user management API routes.
+
+Registers /api/admin/users CRUD endpoints for managing users.
+All endpoints require admin session.
+"""
+
+from functools import wraps
+import os
+import sqlite3
+from typing import Any
+
+from flask import Flask, jsonify, request, session
+from werkzeug.security import generate_password_hash
+
+from shelfmark.config.booklore_settings import (
+    get_booklore_library_options,
+    get_booklore_path_options,
+)
+from shelfmark.config.env import CWA_DB_PATH
+from shelfmark.core.admin_settings_routes import (
+    register_admin_settings_routes,
+    validate_user_settings,
+)
+from shelfmark.core.auth_modes import (
+    AUTH_SOURCE_BUILTIN,
+    AUTH_SOURCE_CWA,
+    AUTH_SOURCE_OIDC,
+    AUTH_SOURCE_PROXY,
+    determine_auth_mode,
+    has_local_password_admin,
+    normalize_auth_source,
+)
+from shelfmark.core.cwa_user_sync import sync_cwa_users_from_rows
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+
+
+def _get_user_edit_capabilities(
+    user: dict[str, Any],
+    security_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Return backend-authored capability flags for the user edit form."""
+    auth_source = normalize_auth_source(
+        user.get("auth_source"),
+        user.get("oidc_subject"),
+    )
+    if security_config is None and auth_source == AUTH_SOURCE_OIDC:
+        security_config = load_config_file("security")
+
+    oidc_use_admin_group = bool((security_config or {}).get("OIDC_USE_ADMIN_GROUP", True))
+    role_managed_by_oidc_group = auth_source == AUTH_SOURCE_OIDC and oidc_use_admin_group
+    can_edit_role = auth_source == AUTH_SOURCE_BUILTIN or (
+        auth_source == AUTH_SOURCE_OIDC and not role_managed_by_oidc_group
+    )
+
+    return {
+        "authSource": auth_source,
+        "canSetPassword": auth_source == AUTH_SOURCE_BUILTIN,
+        "canEditRole": can_edit_role,
+        "canEditEmail": auth_source in {AUTH_SOURCE_BUILTIN, AUTH_SOURCE_PROXY},
+        "canEditDisplayName": auth_source != AUTH_SOURCE_OIDC,
+    }
+
+
+def _get_auth_mode():
+    """Get current auth mode from config."""
+    try:
+        config = load_config_file("security")
+        return determine_auth_mode(
+            config,
+            CWA_DB_PATH,
+            has_local_admin=has_local_password_admin(),
+        )
+    except Exception:
+        return "none"
+
+
+def _require_admin(f):
+    """Decorator to require admin session for admin routes.
+
+    In no-auth mode, everyone has access (is_admin defaults True).
+    In auth-required modes, requires an authenticated session with admin role.
+    """
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        auth_mode = _get_auth_mode()
+        if auth_mode != "none":
+            if "user_id" not in session:
+                return jsonify({"error": "Authentication required"}), 401
+            if not session.get("is_admin", False):
+                return jsonify({"error": "Admin access required"}), 403
+        return f(*args, **kwargs)
+    return decorated
+
+
+def _sanitize_user(user: dict) -> dict:
+    """Remove sensitive fields from user dict before returning to client."""
+    sanitized = dict(user)
+    sanitized.pop("password_hash", None)
+    return sanitized
+
+
+def _oidc_role_management_message(security_config: dict[str, Any]) -> str:
+    admin_group = security_config.get("OIDC_ADMIN_GROUP", "")
+    if admin_group:
+        return (
+            "Admin roles for OIDC users are managed by the "
+            f"'{admin_group}' group in your identity provider"
+        )
+    return (
+        "Disable 'Use Admin Group for Authorization' in security settings "
+        "to manage roles manually"
+    )
+
+
+def _is_user_active(user: dict[str, Any], auth_method: str) -> bool:
+    """Determine whether a user can authenticate in the current auth mode."""
+    source = normalize_auth_source(user.get("auth_source"), user.get("oidc_subject"))
+    if source == AUTH_SOURCE_BUILTIN:
+        return auth_method in (AUTH_SOURCE_BUILTIN, AUTH_SOURCE_OIDC)
+    return source == auth_method
+
+
+def _serialize_user(
+    user: dict[str, Any],
+    auth_method: str,
+    security_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Sanitize and enrich a user payload for API responses."""
+    payload = _sanitize_user(user)
+    payload["auth_source"] = normalize_auth_source(
+        payload.get("auth_source"),
+        payload.get("oidc_subject"),
+    )
+    payload["is_active"] = _is_user_active(payload, auth_method)
+    payload["edit_capabilities"] = _get_user_edit_capabilities(
+        payload,
+        security_config=security_config,
+    )
+    return payload
+
+
+def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]:
+    """Sync all users from the Calibre-Web database into users.db."""
+    if not CWA_DB_PATH or not CWA_DB_PATH.exists():
+        raise FileNotFoundError("Calibre-Web database is not available")
+
+    db_path = os.fspath(CWA_DB_PATH)
+    db_uri = f"file:{db_path}?mode=ro&immutable=1"
+    conn = sqlite3.connect(db_uri, uri=True)
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT name, role, email FROM user")
+        rows = cur.fetchall()
+    finally:
+        conn.close()
+
+    return sync_cwa_users_from_rows(user_db, rows)
+
+
+def register_admin_routes(app: Flask, user_db: UserDB) -> None:
+    """Register admin user management routes on the Flask app."""
+
+    @app.route("/api/admin/users", methods=["GET"])
+    @_require_admin
+    def admin_list_users():
+        """List all users."""
+        users = user_db.list_users()
+        auth_mode = _get_auth_mode()
+        security_config = load_config_file("security")
+        return jsonify([
+            _serialize_user(u, auth_mode, security_config=security_config)
+            for u in users
+        ])
+
+    @app.route("/api/admin/users", methods=["POST"])
+    @_require_admin
+    def admin_create_user():
+        """Create a new user with password authentication."""
+        data = request.get_json() or {}
+        auth_mode = _get_auth_mode()
+
+        username = (data.get("username") or "").strip()
+        password = data.get("password", "")
+        email = (data.get("email") or "").strip() or None
+        display_name = (data.get("display_name") or "").strip() or None
+        role = data.get("role", "user")
+
+        if auth_mode in {AUTH_SOURCE_PROXY, AUTH_SOURCE_CWA}:
+            return jsonify({
+                "error": "Local user creation is disabled in this authentication mode",
+                "message": (
+                    "Users are provisioned by your external authentication source. "
+                    "Switch to builtin or OIDC mode to create local users."
+                ),
+            }), 400
+
+        if not username:
+            return jsonify({"error": "Username is required"}), 400
+        if not password or len(password) < 4:
+            return jsonify({"error": "Password must be at least 4 characters"}), 400
+        if role not in ("admin", "user"):
+            return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
+
+        # First user is always admin
+        if not user_db.list_users():
+            role = "admin"
+
+        # Check if username already exists
+        if user_db.get_user(username=username):
+            return jsonify({"error": "Username already exists"}), 409
+
+        password_hash = generate_password_hash(password)
+        try:
+            user = user_db.create_user(
+                username=username,
+                password_hash=password_hash,
+                email=email,
+                display_name=display_name,
+                auth_source=AUTH_SOURCE_BUILTIN,
+                role=role,
+            )
+        except ValueError:
+            return jsonify({"error": "Username already exists"}), 409
+        logger.info(
+            "Shelfmark user created "
+            f"(source=manual_admin_create, created_by={session.get('user_id', 'unknown')}, "
+            f"username={username}, role={role}, auth_source={AUTH_SOURCE_BUILTIN})"
+        )
+        return jsonify(
+            _serialize_user(
+                user,
+                _get_auth_mode(),
+                security_config=load_config_file("security"),
+            )
+        ), 201
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["GET"])
+    @_require_admin
+    def admin_get_user(user_id):
+        """Get a user by ID with their settings."""
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        result = _serialize_user(
+            user,
+            _get_auth_mode(),
+            security_config=load_config_file("security"),
+        )
+        result["settings"] = user_db.get_user_settings(user_id)
+        return jsonify(result)
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["PUT"])
+    @_require_admin
+    def admin_update_user(user_id):
+        """Update user fields and/or settings."""
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        data = request.get_json() or {}
+        security_config = load_config_file("security")
+        auth_source = normalize_auth_source(
+            user.get("auth_source"),
+            user.get("oidc_subject"),
+        )
+        capabilities = _get_user_edit_capabilities(user, security_config=security_config)
+
+        # Handle optional password update
+        password = data.get("password", "")
+        if password:
+            if not capabilities["canSetPassword"]:
+                return jsonify({
+                    "error": f"Cannot set password for {auth_source.upper()} users",
+                    "message": "Password authentication is only available for local users.",
+                }), 400
+            if len(password) < 4:
+                return jsonify({"error": "Password must be at least 4 characters"}), 400
+            user_db.update_user(user_id, password_hash=generate_password_hash(password))
+
+        # Update user fields
+        user_fields = {}
+        for field in ("role", "email", "display_name"):
+            if field in data:
+                user_fields[field] = data[field]
+
+        if "role" in user_fields and user_fields["role"] not in ("admin", "user"):
+            return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
+
+        role_changed = "role" in user_fields and user_fields["role"] != user.get("role")
+        email_changed = "email" in user_fields and user_fields["email"] != user.get("email")
+        display_name_changed = (
+            "display_name" in user_fields
+            and user_fields["display_name"] != user.get("display_name")
+        )
+
+        if role_changed and not capabilities["canEditRole"]:
+            if auth_source == AUTH_SOURCE_OIDC:
+                return jsonify({
+                    "error": "Cannot change role for OIDC user when group-based authorization is enabled",
+                    "message": _oidc_role_management_message(security_config),
+                }), 400
+
+            return jsonify({
+                "error": f"Cannot change role for {auth_source.upper()} users",
+                "message": "Role is managed by the external authentication source.",
+            }), 400
+
+        if email_changed and not capabilities["canEditEmail"]:
+            if auth_source == AUTH_SOURCE_CWA:
+                return jsonify({
+                    "error": "Cannot change email for CWA users",
+                    "message": "Email is synced from Calibre-Web.",
+                }), 400
+
+            return jsonify({
+                "error": "Cannot change email for OIDC users",
+                "message": "Email is managed by your identity provider.",
+            }), 400
+
+        if display_name_changed and not capabilities["canEditDisplayName"]:
+            return jsonify({
+                "error": "Cannot change display name for OIDC users",
+                "message": "Display name is managed by your identity provider.",
+            }), 400
+
+        # Allow demoting the last admin account.
+        # Auth mode resolution automatically falls back to "none" when no
+        # local password admin remains.
+
+        # Avoid unnecessary writes for no-op field updates.
+        for field in ("role", "email", "display_name"):
+            if field in user_fields and user_fields[field] == user.get(field):
+                user_fields.pop(field)
+
+        if user_fields:
+            user_db.update_user(user_id, **user_fields)
+
+        # Update per-user settings
+        if "settings" in data:
+            if not isinstance(data["settings"], dict):
+                return jsonify({"error": "Settings must be an object"}), 400
+
+            validated_settings, validation_errors = validate_user_settings(data["settings"])
+            if validation_errors:
+                return jsonify({
+                    "error": "Invalid settings payload",
+                    "details": validation_errors,
+                }), 400
+
+            user_db.set_user_settings(user_id, validated_settings)
+            # Ensure runtime reads see updated per-user overrides immediately.
+            try:
+                from shelfmark.core.config import config as app_config
+                app_config.refresh()
+            except Exception:
+                pass
+
+        updated = user_db.get_user(user_id=user_id)
+        result = _serialize_user(
+            updated,
+            _get_auth_mode(),
+            security_config=security_config,
+        )
+        result["settings"] = user_db.get_user_settings(user_id)
+        logger.info(f"Admin updated user {user_id}")
+        return jsonify(result)
+
+    @app.route("/api/admin/users/sync-cwa", methods=["POST"])
+    @_require_admin
+    def admin_sync_cwa_users():
+        """Manually sync users from Calibre-Web into users.db."""
+        auth_mode = _get_auth_mode()
+        if auth_mode != AUTH_SOURCE_CWA:
+            return jsonify({
+                "error": "CWA sync is only available when CWA authentication is enabled",
+            }), 400
+
+        try:
+            summary = _sync_all_cwa_users(user_db)
+        except FileNotFoundError:
+            return jsonify({
+                "error": "Calibre-Web database is not available",
+                "message": "Verify app.db is mounted and readable at /auth/app.db.",
+            }), 503
+        except Exception as exc:
+            logger.error(f"Failed to sync CWA users: {exc}")
+            return jsonify({
+                "error": "Failed to sync users from Calibre-Web",
+            }), 500
+
+        message = (
+            f"Synced {summary['total']} CWA users "
+            f"({summary['created']} created, {summary['updated']} updated, "
+            f"{summary.get('deleted', 0)} deleted)."
+        )
+        logger.info(message)
+        return jsonify({
+            "success": True,
+            "message": message,
+            **summary,
+        })
+
+    register_admin_settings_routes(app, user_db, _require_admin)
+
+    @app.route("/api/admin/users/<int:user_id>", methods=["DELETE"])
+    @_require_admin
+    def admin_delete_user(user_id):
+        """Delete a user."""
+        # Prevent self-deletion
+        if session.get("db_user_id") == user_id:
+            return jsonify({"error": "Cannot delete your own account"}), 400
+
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        auth_mode = _get_auth_mode()
+        auth_source = normalize_auth_source(
+            user.get("auth_source"),
+            user.get("oidc_subject"),
+        )
+        if auth_source == AUTH_SOURCE_CWA and auth_source == auth_mode:
+            return jsonify({
+                "error": f"Cannot delete active {auth_source.upper()} users",
+                "message": f"{auth_source.upper()} users are automatically re-provisioned on login.",
+            }), 400
+
+        # Allow deleting the last local admin account.
+        # Auth mode resolution automatically falls back to "none" when no
+        # local password admin remains.
+
+        user_db.delete_user(user_id)
+        logger.info(f"Admin deleted user {user_id}: {user['username']}")
+        return jsonify({"success": True})

shelfmark/core/admin_settings_routes.py

@@ -0,0 +1,230 @@
+"""Admin settings-introspection routes and settings validation helpers."""
+
+from typing import Any, Callable
+
+from flask import Flask, jsonify, request
+
+from shelfmark.config.notifications_settings import (
+    build_notification_test_result,
+    is_valid_notification_url,
+    normalize_notification_routes,
+)
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_settings_overrides import (
+    build_user_preferences_payload as _build_user_preferences_payload,
+    get_ordered_user_overridable_fields as _get_ordered_user_overridable_fields,
+    get_settings_registry as _get_settings_registry,
+)
+from shelfmark.core.user_db import UserDB
+from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
+
+
+def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
+    settings_registry = _get_settings_registry()
+    field_map = settings_registry.get_settings_field_map()
+    overridable_map = settings_registry.get_user_overridable_fields()
+
+    valid: dict[str, Any] = {}
+    errors: list[str] = []
+    for key, value in settings.items():
+        if key not in field_map:
+            errors.append(f"Unknown setting: {key}")
+        elif key not in overridable_map:
+            errors.append(f"Setting not user-overridable: {key}")
+        else:
+            # null means "clear the per-user override; use global default"
+            if value is None:
+                valid[key] = None
+                continue
+
+            if key in {"REQUEST_POLICY_DEFAULT_EBOOK", "REQUEST_POLICY_DEFAULT_AUDIOBOOK"}:
+                if parse_policy_mode(value) is None:
+                    errors.append(f"Invalid policy mode for {key}: {value}")
+                    continue
+
+            if key == "REQUEST_POLICY_RULES":
+                normalized_rules, rule_errors = validate_policy_rules(value)
+                if rule_errors:
+                    errors.extend(rule_errors)
+                    continue
+                valid[key] = normalized_rules
+                continue
+
+            if key == "USER_NOTIFICATION_ROUTES":
+                normalized_routes = normalize_notification_routes(value)
+                invalid_count = sum(
+                    1
+                    for row in normalized_routes
+                    if row.get("url") and not is_valid_notification_url(str(row.get("url")))
+                )
+                if invalid_count:
+                    errors.append(
+                        (
+                            f"Invalid value for {key}: found {invalid_count} invalid URL(s). "
+                            "Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
+                        )
+                    )
+                    continue
+                valid[key] = normalized_routes
+                continue
+
+            valid[key] = value
+
+    return valid, errors
+
+
+def build_user_notification_test_response(
+    *,
+    user_id: int,
+    payload: Any,
+) -> tuple[dict[str, Any], int]:
+    from shelfmark.core.config import config as app_config
+
+    routes_input = app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=user_id)
+    if isinstance(payload, dict):
+        if "USER_NOTIFICATION_ROUTES" in payload:
+            routes_input = payload.get("USER_NOTIFICATION_ROUTES")
+        elif "routes" in payload:
+            routes_input = payload.get("routes")
+
+    result = build_notification_test_result(routes_input, scope_label="personal")
+    status_code = 200 if result.get("success", False) else 400
+    return result, status_code
+
+
+def register_admin_settings_routes(
+    app: Flask,
+    user_db: UserDB,
+    require_admin: Callable[[Callable[..., Any]], Callable[..., Any]],
+) -> None:
+    @app.route("/api/admin/download-defaults", methods=["GET"])
+    @require_admin
+    def admin_download_defaults():
+        config = load_config_file("downloads")
+        defaults = {
+            key: ("" if (value := config.get(key, field.default)) is None else value)
+            for key, field in _get_ordered_user_overridable_fields("downloads")
+        }
+
+        security_config = load_config_file("security")
+        defaults["OIDC_ADMIN_GROUP"] = security_config.get("OIDC_ADMIN_GROUP", "")
+        defaults["OIDC_USE_ADMIN_GROUP"] = security_config.get("OIDC_USE_ADMIN_GROUP", True)
+        defaults["OIDC_AUTO_PROVISION"] = security_config.get("OIDC_AUTO_PROVISION", True)
+        return jsonify(defaults)
+
+    @app.route("/api/admin/booklore-options", methods=["GET"])
+    @require_admin
+    def admin_booklore_options():
+        from shelfmark.core import admin_routes
+
+        return jsonify({
+            "libraries": admin_routes.get_booklore_library_options(),
+            "paths": admin_routes.get_booklore_path_options(),
+        })
+
+    @app.route("/api/admin/users/<int:user_id>/delivery-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_delivery_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "downloads")
+        except ValueError:
+            return jsonify({"error": "Downloads settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/notification-preferences", methods=["GET"])
+    @require_admin
+    def admin_get_notification_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        try:
+            payload = _build_user_preferences_payload(user_db, user_id, "notifications")
+        except ValueError:
+            return jsonify({"error": "Notifications settings tab not found"}), 500
+
+        return jsonify(payload)
+
+    @app.route("/api/admin/users/<int:user_id>/notification-preferences/test", methods=["POST"])
+    @require_admin
+    def admin_test_notification_preferences(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        payload = request.get_json(silent=True)
+        result, status_code = build_user_notification_test_response(
+            user_id=user_id,
+            payload=payload,
+        )
+        return jsonify(result), status_code
+
+    @app.route("/api/admin/settings/overrides-summary", methods=["GET"])
+    @require_admin
+    def admin_settings_overrides_summary():
+        settings_registry = _get_settings_registry()
+
+        tab_name = (request.args.get("tab") or "downloads").strip()
+        if not settings_registry.get_settings_tab(tab_name):
+            return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404
+
+        overridable_keys = list(settings_registry.get_user_overridable_fields(tab_name=tab_name))
+        keys_payload: dict[str, dict[str, Any]] = {}
+
+        for user_record in user_db.list_users():
+            user_settings = user_db.get_user_settings(user_record["id"])
+            if not isinstance(user_settings, dict):
+                continue
+
+            for key in overridable_keys:
+                if key not in user_settings or user_settings[key] is None:
+                    continue
+                entry = keys_payload.setdefault(key, {"count": 0, "users": []})
+                entry["users"].append({
+                    "userId": user_record["id"],
+                    "username": user_record["username"],
+                    "value": user_settings[key],
+                })
+
+        for summary in keys_payload.values():
+            summary["count"] = len(summary["users"])
+
+        return jsonify({"tab": tab_name, "keys": keys_payload})
+
+    @app.route("/api/admin/users/<int:user_id>/effective-settings", methods=["GET"])
+    @require_admin
+    def admin_get_effective_settings(user_id):
+        user = user_db.get_user(user_id=user_id)
+        if not user:
+            return jsonify({"error": "User not found"}), 404
+
+        from shelfmark.core.config import config as app_config
+        from shelfmark.core.settings_registry import is_value_from_env
+
+        field_map = _get_settings_registry().get_user_overridable_fields()
+        user_settings = user_db.get_user_settings(user_id)
+        tab_config_cache: dict[str, dict[str, Any]] = {}
+        effective: dict[str, dict[str, Any]] = {}
+
+        for key, (field, tab_name) in sorted(field_map.items()):
+            value = app_config.get(key, field.default, user_id=user_id)
+            source = "default"
+
+            if field.env_supported and is_value_from_env(field):
+                source = "env_var"
+            elif key in user_settings and user_settings[key] is not None:
+                source = "user_override"
+                value = user_settings[key]
+            else:
+                tab_config = tab_config_cache.setdefault(tab_name, load_config_file(tab_name))
+                if key in tab_config:
+                    source = "global_config"
+
+            effective[key] = {"value": value, "source": source}
+
+        return jsonify(effective)

shelfmark/core/auth_modes.py

@@ -0,0 +1,126 @@
+"""Authentication mode, auth-source normalization, and admin access policy helpers."""
+
+import os
+from typing import Any, Mapping
+
+AUTH_SOURCE_BUILTIN = "builtin"
+AUTH_SOURCE_OIDC = "oidc"
+AUTH_SOURCE_PROXY = "proxy"
+AUTH_SOURCE_CWA = "cwa"
+AUTH_SOURCES = (
+    AUTH_SOURCE_BUILTIN,
+    AUTH_SOURCE_OIDC,
+    AUTH_SOURCE_PROXY,
+    AUTH_SOURCE_CWA,
+)
+AUTH_SOURCE_SET = frozenset(AUTH_SOURCES)
+_ALWAYS_ADMIN_SETTINGS_TABS = frozenset({"security", "users"})
+
+
+def has_local_password_admin(user_db: Any | None = None) -> bool:
+    """Return True when at least one local admin with a password exists."""
+    try:
+        db = user_db
+        if db is None:
+            from shelfmark.core.user_db import UserDB
+
+            config_root = os.environ.get("CONFIG_DIR", "/config")
+            db = UserDB(os.path.join(config_root, "users.db"))
+            db.initialize()
+
+        return any(
+            user.get("password_hash") and user.get("role") == "admin"
+            for user in db.list_users()
+        )
+    except Exception:
+        return False
+
+
+def normalize_auth_source(
+    source: Any,
+    oidc_subject: Any = None,
+) -> str:
+    """Resolve a stable auth source value from persisted fields."""
+    normalized = str(source or "").strip().lower()
+    if normalized in AUTH_SOURCE_SET:
+        return normalized
+    if oidc_subject:
+        return AUTH_SOURCE_OIDC
+    return AUTH_SOURCE_BUILTIN
+
+
+def determine_auth_mode(
+    security_config: Mapping[str, Any],
+    cwa_db_path: Any | None,
+    *,
+    has_local_admin: bool = True,
+) -> str:
+    """Determine active auth mode from security config and runtime prerequisites."""
+    auth_mode = security_config.get("AUTH_METHOD", "none")
+
+    if auth_mode == AUTH_SOURCE_CWA and cwa_db_path:
+        return AUTH_SOURCE_CWA
+
+    if auth_mode == AUTH_SOURCE_BUILTIN and has_local_admin:
+        return AUTH_SOURCE_BUILTIN
+
+    if auth_mode == AUTH_SOURCE_PROXY and security_config.get("PROXY_AUTH_USER_HEADER"):
+        return AUTH_SOURCE_PROXY
+
+    if (
+        auth_mode == AUTH_SOURCE_OIDC
+        and has_local_admin
+        and security_config.get("OIDC_DISCOVERY_URL")
+        and security_config.get("OIDC_CLIENT_ID")
+    ):
+        return AUTH_SOURCE_OIDC
+
+    return "none"
+
+
+def is_settings_or_onboarding_path(path: str) -> bool:
+    """Return True when request path targets protected admin settings routes."""
+    return path.startswith("/api/settings") or path.startswith("/api/onboarding")
+
+
+def get_settings_tab_from_path(path: str) -> str | None:
+    """Extract tab name from /api/settings/<tab>[...] paths."""
+    if not path.startswith("/api/settings/"):
+        return None
+
+    suffix = path[len("/api/settings/"):]
+    if not suffix:
+        return None
+
+    return suffix.split("/", 1)[0] or None
+
+
+def should_restrict_settings_to_admin(
+    _users_config: Mapping[str, Any],
+) -> bool:
+    """Settings/onboarding is always admin-only."""
+    return True
+
+
+def requires_admin_for_settings_access(
+    path: str,
+    users_config: Mapping[str, Any],
+) -> bool:
+    """Return whether this settings/onboarding request requires admin privileges."""
+    tab_name = get_settings_tab_from_path(path)
+    if tab_name in _ALWAYS_ADMIN_SETTINGS_TABS:
+        return True
+
+    return should_restrict_settings_to_admin(users_config)
+
+
+def get_auth_check_admin_status(
+    _auth_mode: str,
+    _users_config: Mapping[str, Any],
+    session_data: Mapping[str, Any],
+) -> bool:
+    """Resolve /api/auth/check `is_admin` as the session's real admin role."""
+    if "user_id" not in session_data:
+        return False
+
+    return bool(session_data.get("is_admin", False))

shelfmark/core/cache.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 from functools import wraps
 from typing import Any, Callable, Dict, Optional, TypeVar
 
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -24,24 +24,13 @@ class CacheService:
     """Thread-safe in-memory cache with TTL support."""
 
     def __init__(self, max_size: int = 1000):
-        """Initialize cache service.
-
-        Args:
-            max_size: Maximum number of entries before oldest are evicted.
-        """
+        """Initialize cache with max_size entries before eviction."""
         self._cache: Dict[str, CacheEntry] = {}
         self._lock = threading.Lock()
         self._max_size = max_size
 
     def get(self, key: str) -> Optional[Any]:
-        """Get cached value if not expired.
-
-        Args:
-            key: Cache key to retrieve.
-
-        Returns:
-            Cached value or None if not found/expired.
-        """
+        """Get cached value if not expired."""
         with self._lock:
             entry = self._cache.get(key)
             if entry is None:
@@ -54,13 +43,7 @@ class CacheService:
             return entry.value
 
     def set(self, key: str, value: Any, ttl: int) -> None:
-        """Cache value with TTL.
-
-        Args:
-            key: Cache key.
-            value: Value to cache.
-            ttl: Time to live in seconds.
-        """
+        """Cache value with TTL in seconds."""
         with self._lock:
             # Evict oldest entries if at capacity
             if len(self._cache) >= self._max_size:
@@ -72,14 +55,7 @@ class CacheService:
             )
 
     def invalidate(self, key: str) -> bool:
-        """Remove specific cache entry.
-
-        Args:
-            key: Cache key to remove.
-
-        Returns:
-            True if entry was removed, False if not found.
-        """
+        """Remove specific cache entry. Returns True if found."""
         with self._lock:
             if key in self._cache:
                 del self._cache[key]
@@ -92,11 +68,7 @@ class CacheService:
             self._cache.clear()
 
     def cleanup_expired(self) -> int:
-        """Remove all expired entries.
-
-        Returns:
-            Number of entries removed.
-        """
+        """Remove all expired entries. Returns count removed."""
         with self._lock:
             now = time.time()
             expired_keys = [
@@ -108,10 +80,7 @@ class CacheService:
             return len(expired_keys)
 
     def _evict_oldest(self) -> None:
-        """Evict oldest entries (by expiration time) to make room.
-
-        Called with lock held.
-        """
+        """Evict ~10% of oldest entries. Called with lock held."""
         if not self._cache:
             return
 
@@ -126,11 +95,7 @@ class CacheService:
             del self._cache[key]
 
     def stats(self) -> Dict[str, int]:
-        """Get cache statistics.
-
-        Returns:
-            Dict with size and max_size.
-        """
+        """Get cache statistics (size, max_size)."""
         with self._lock:
             return {
                 "size": len(self._cache),
@@ -148,15 +113,7 @@ def get_metadata_cache() -> CacheService:
 
 
 def cache_key(*args, **kwargs) -> str:
-    """Generate cache key from arguments.
-
-    Args:
-        *args: Positional arguments to include in key.
-        **kwargs: Keyword arguments to include in key.
-
-    Returns:
-        String cache key.
-    """
+    """Generate cache key from arguments."""
     parts = [str(arg) for arg in args]
     parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
     return ":".join(parts)
@@ -168,23 +125,12 @@ def cacheable(
     ttl_default: int = 300,
     key_prefix: str = ""
 ):
-    """Decorator for caching function results.
-
-    Args:
-        ttl: Static time to live in seconds (use this OR ttl_key, not both).
-        ttl_key: Config key to read TTL from (e.g., "METADATA_CACHE_SEARCH_TTL").
-        ttl_default: Default TTL if ttl_key not found in config.
-        key_prefix: Optional prefix for cache keys.
-
-    Examples:
-        @cacheable(ttl=300, key_prefix="hardcover:search")  # Static TTL
-        @cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", key_prefix="hardcover:search")  # Dynamic TTL
-    """
+    """Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
     def decorator(func: Callable[..., T]) -> Callable[..., T]:
         @wraps(func)
         def wrapper(*args, **kwargs) -> T:
             # Check if metadata caching is enabled
-            from cwa_book_downloader.core.config import config
+            from shelfmark.core.config import config
 
             if not config.get("METADATA_CACHE_ENABLED", True):
                 # Caching disabled, execute function directly
@@ -211,11 +157,9 @@ def cacheable(
             # Check cache
             cached = _metadata_cache.get(key)
             if cached is not None:
-                logger.debug(f"Cache hit: {key}")
                 return cached
 
             # Execute function and cache result
-            logger.debug(f"Cache miss: {key}")
             result = func(*args, **kwargs)
 
             # Only cache non-None results

shelfmark/core/config.py

@@ -1,18 +1,21 @@
 """Configuration singleton with ENV > config file > default resolution."""
 
+import os
+import sqlite3
 from threading import Lock
 from typing import Any, Dict, Optional
 
 # Import lazily to avoid circular imports
 _registry_module = None
 _env_module = None
+_user_db_module = None
 
 
 def _get_registry():
     """Lazy import of settings registry to avoid circular imports."""
     global _registry_module
     if _registry_module is None:
-        from cwa_book_downloader.core import settings_registry
+        from shelfmark.core import settings_registry
         _registry_module = settings_registry
     return _registry_module
 
@@ -21,11 +24,20 @@ def _get_env():
     """Lazy import of env module for fallback values."""
     global _env_module
     if _env_module is None:
-        from cwa_book_downloader.config import env
+        from shelfmark.config import env
         _env_module = env
     return _env_module
 
 
+def _get_user_db_module():
+    """Lazy import of user DB module to avoid optional dependency loops."""
+    global _user_db_module
+    if _user_db_module is None:
+        from shelfmark.core.user_db import UserDB
+        _user_db_module = UserDB
+    return _user_db_module
+
+
 class Config:
     """
     Dynamic configuration singleton that provides live settings access.
@@ -36,7 +48,6 @@ class Config:
 
     _instance: Optional['Config'] = None
     _lock = Lock()
-
     def __new__(cls) -> 'Config':
         if cls._instance is None:
             with cls._lock:
@@ -51,6 +62,10 @@ class Config:
         self._cache: Dict[str, Any] = {}
         self._field_map: Dict[str, tuple] = {}  # key -> (field, tab_name)
         self._cache_lock = Lock()
+        self._user_settings_cache: Dict[int, Dict[str, Any]] = {}
+        self._user_settings_cache_lock = Lock()
+        self._user_db = None
+        self._user_db_load_attempted = False
         self._initialized = True
         self._loaded = False
 
@@ -65,11 +80,13 @@ class Config:
 
     def _load_settings(self) -> None:
         """Load all settings from the registry."""
-        # Ensure all plugin settings are registered before loading
-        # This handles cases where config is accessed before plugins are imported
+        # Ensure all settings modules are imported before loading
+        # This handles cases where config is accessed before settings are registered
         try:
-            import cwa_book_downloader.release_sources  # noqa: F401
-            import cwa_book_downloader.metadata_providers  # noqa: F401
+            import shelfmark.config.settings  # noqa: F401 - main app settings
+            import shelfmark.config.notifications_settings  # noqa: F401 - notifications settings
+            import shelfmark.release_sources  # noqa: F401 - plugin settings
+            import shelfmark.metadata_providers  # noqa: F401 - plugin settings
         except ImportError:
             pass
 
@@ -110,19 +127,85 @@ class Config:
         with self._cache_lock:
             self._loaded = False
             self._load_settings()
+        with self._user_settings_cache_lock:
+            self._user_settings_cache.clear()
+        self._user_db = None
+        self._user_db_load_attempted = False
 
-    def get(self, key: str, default: Any = None) -> Any:
+    def _get_user_db(self):
+        """Get or initialize a UserDB handle if available."""
+        if self._user_db is not None:
+            return self._user_db
+        if self._user_db_load_attempted:
+            return None
+
+        self._user_db_load_attempted = True
+        try:
+            user_db_cls = _get_user_db_module()
+            db_path = os.path.join(os.environ.get("CONFIG_DIR", "/config"), "users.db")
+            user_db = user_db_cls(db_path)
+            user_db.initialize()
+            self._user_db = user_db
+            return self._user_db
+        except Exception:
+            # Multi-user support is optional; fall back to global config when unavailable.
+            return None
+
+    def _get_user_settings(self, user_id: int) -> Dict[str, Any]:
+        """Get cached per-user settings from user DB."""
+        with self._user_settings_cache_lock:
+            if user_id in self._user_settings_cache:
+                return self._user_settings_cache[user_id]
+
+        user_db = self._get_user_db()
+        if user_db is None:
+            return {}
+
+        try:
+            settings = user_db.get_user_settings(user_id)
+        except (sqlite3.OperationalError, OSError, ValueError, TypeError):
+            return {}
+
+        if not isinstance(settings, dict):
+            settings = {}
+
+        with self._user_settings_cache_lock:
+            self._user_settings_cache[user_id] = settings
+        return settings
+
+    def _get_user_override(self, user_id: int, key: str) -> Any:
+        """Get a user override for a specific key."""
+        user_settings = self._get_user_settings(user_id)
+        return user_settings.get(key)
+
+    def get(self, key: str, default: Any = None, user_id: Optional[int] = None) -> Any:
         """
         Get a setting value by key.
 
         Args:
             key: The setting key (e.g., 'MAX_RETRY')
             default: Default value if setting not found
+            user_id: Optional DB user ID for per-user setting overrides
 
         Returns:
             The setting value, or default if not found
         """
         self._ensure_loaded()
+
+        if key in self._field_map:
+            field, _ = self._field_map[key]
+            registry = _get_registry()
+
+            # Deployment-level ENV values always win.
+            if field.env_supported and registry.is_value_from_env(field):
+                return self._cache.get(key, default)
+
+            # User overrides are only available for explicitly overridable fields.
+            if user_id is not None and getattr(field, "user_overridable", False):
+                user_value = self._get_user_override(user_id, key)
+                if user_value is not None:
+                    return user_value
+
         return self._cache.get(key, default)
 
     def __getattr__(self, name: str) -> Any:

shelfmark/core/cwa_user_sync.py

@@ -0,0 +1,94 @@
+"""Helpers for provisioning and syncing Calibre-Web users into users.db."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from shelfmark.core.auth_modes import AUTH_SOURCE_CWA, normalize_auth_source
+from shelfmark.core.external_user_linking import upsert_external_user
+from shelfmark.core.user_db import UserDB
+
+_CWA_ALIAS_SUFFIX = "__cwa"
+
+
+def _normalize_email(value: Any) -> str | None:
+    if value is None:
+        return None
+    email = str(value).strip()
+    return email or None
+
+
+def upsert_cwa_user(
+    user_db: UserDB,
+    cwa_username: str,
+    cwa_email: str | None,
+    role: str,
+    context: str | None = None,
+) -> tuple[dict[str, Any], str]:
+    """Create/update a CWA-backed user with collision-safe matching."""
+    normalized_email = _normalize_email(cwa_email)
+    collision_strategy = "alias" if normalized_email else "takeover"
+    user, action = upsert_external_user(
+        user_db,
+        auth_source="cwa",
+        username=cwa_username,
+        email=normalized_email,
+        role=role,
+        allow_email_link=True,
+        collision_strategy=collision_strategy,
+        alias_suffix=_CWA_ALIAS_SUFFIX,
+        context=context,
+    )
+    if user is None:
+        raise RuntimeError("Unexpected CWA user sync result: no user returned")
+    return user, action
+
+
+def sync_cwa_users_from_rows(
+    user_db: UserDB,
+    rows: Iterable[tuple[Any, Any, Any]],
+) -> dict[str, int]:
+    """Sync CWA users from raw `(name, role_flags, email)` rows."""
+    created = 0
+    updated = 0
+    active_cwa_user_ids: set[int] = set()
+    for username, role_flags, email in rows:
+        normalized_username = str(username or "").strip()
+        if not normalized_username:
+            continue
+
+        role = "admin" if (int(role_flags or 0) & 1) == 1 else "user"
+        user, action = upsert_cwa_user(
+            user_db,
+            cwa_username=normalized_username,
+            cwa_email=_normalize_email(email),
+            role=role,
+            context="cwa_manual_sync",
+        )
+        active_cwa_user_ids.add(int(user["id"]))
+        if action == "created":
+            created += 1
+        else:
+            updated += 1
+
+    deleted = 0
+    for existing_user in user_db.list_users():
+        if normalize_auth_source(
+            existing_user.get("auth_source"),
+            existing_user.get("oidc_subject"),
+        ) != AUTH_SOURCE_CWA:
+            continue
+
+        existing_id = int(existing_user.get("id") or 0)
+        if existing_id in active_cwa_user_ids:
+            continue
+
+        user_db.delete_user(existing_id)
+        deleted += 1
+
+    return {
+        "created": created,
+        "updated": updated,
+        "deleted": deleted,
+        "total": created + updated,
+    }

shelfmark/core/external_user_linking.py

@@ -0,0 +1,283 @@
+"""Shared external identity matching and provisioning helpers."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Literal
+
+from shelfmark.core.auth_modes import normalize_auth_source
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.user_db import UserDB
+
+UNSET = object()
+
+CollisionStrategy = Literal["takeover", "suffix", "alias"]
+MatchReason = Literal[
+    "subject_match",
+    "existing_source_username_match",
+    "unique_email_match",
+]
+
+logger = setup_logger(__name__)
+
+
+def _normalize_username(value: Any) -> str:
+    return str(value or "").strip()
+
+
+def _normalize_email(value: Any) -> str | None:
+    if value is None:
+        return None
+    email = str(value).strip()
+    return email or None
+
+
+def _normalize_display_name(value: Any) -> str | None:
+    if value is None:
+        return None
+    name = str(value).strip()
+    return name or None
+
+
+def _email_key(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def _normalize_role(value: Any) -> str:
+    return "admin" if str(value or "").strip().lower() == "admin" else "user"
+
+
+def _get_by_subject(user_db: UserDB, subject_field: str | None, subject: str | None) -> dict[str, Any] | None:
+    if not subject_field or not subject:
+        return None
+    if subject_field == "oidc_subject":
+        return user_db.get_user(oidc_subject=subject)
+    return None
+
+
+def find_unique_user_by_email(user_db: UserDB, email: str | None) -> dict[str, Any] | None:
+    key = _email_key(_normalize_email(email))
+    if not key:
+        return None
+
+    matches = [u for u in user_db.list_users() if _email_key(u.get("email")) == key]
+    return matches[0] if len(matches) == 1 else None
+
+
+def find_external_user_match(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    username: str,
+    email: str | None,
+    subject_field: str | None = None,
+    subject: str | None = None,
+    allow_email_link: bool = False,
+) -> tuple[dict[str, Any] | None, MatchReason | None]:
+    """Find an existing local user that should be linked to an external identity."""
+    normalized_username = _normalize_username(username)
+    normalized_email = _normalize_email(email)
+
+    by_subject = _get_by_subject(user_db, subject_field, subject)
+    if by_subject is not None:
+        return by_subject, "subject_match"
+
+    by_username = user_db.get_user(username=normalized_username)
+    if by_username and normalize_auth_source(
+        by_username.get("auth_source"),
+        by_username.get("oidc_subject"),
+    ) == auth_source:
+        return by_username, "existing_source_username_match"
+
+    if allow_email_link:
+        return find_unique_user_by_email(user_db, normalized_email), "unique_email_match"
+    return None, None
+
+
+def _build_updates(
+    *,
+    auth_source: str,
+    role: str,
+    sync_role: bool,
+    email: str | None | object,
+    display_name: str | None | object,
+    subject_field: str | None,
+    subject: str | None,
+) -> dict[str, Any]:
+    updates: dict[str, Any] = {"auth_source": auth_source}
+    if sync_role:
+        updates["role"] = _normalize_role(role)
+    if email is not UNSET:
+        updates["email"] = _normalize_email(email)
+    if display_name is not UNSET:
+        updates["display_name"] = _normalize_display_name(display_name)
+    if subject_field == "oidc_subject" and subject:
+        updates["oidc_subject"] = subject
+    return updates
+
+
+def _next_suffix_username(user_db: UserDB, base_username: str) -> str:
+    candidate = base_username
+    suffix = 1
+    while user_db.get_user(username=candidate):
+        candidate = f"{base_username}_{suffix}"
+        suffix += 1
+    return candidate
+
+
+def _find_existing_alias_user(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    alias_base: str,
+) -> dict[str, Any] | None:
+    pattern = re.compile(rf"^{re.escape(alias_base)}(?:_\d+)?$")
+    candidates = [
+        user for user in user_db.list_users()
+        if pattern.match(str(user.get("username") or ""))
+        and normalize_auth_source(user.get("auth_source"), user.get("oidc_subject")) == auth_source
+    ]
+    if not candidates:
+        return None
+    return sorted(candidates, key=lambda user: int(user.get("id") or 0))[0]
+
+
+def _resolve_create_username(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    requested_username: str,
+    strategy: CollisionStrategy,
+    alias_suffix: str,
+) -> tuple[str | None, dict[str, Any] | None, str]:
+    existing = user_db.get_user(username=requested_username)
+    if not existing:
+        return requested_username, None, "new_username_available"
+
+    if strategy == "takeover":
+        return None, existing, "username_collision_takeover"
+
+    if strategy == "suffix":
+        return _next_suffix_username(user_db, requested_username), None, "username_collision_suffix"
+
+    alias_base = f"{requested_username}{alias_suffix}"
+    alias_existing = _find_existing_alias_user(
+        user_db,
+        auth_source=auth_source,
+        alias_base=alias_base,
+    )
+    if alias_existing is not None:
+        return None, alias_existing, "reuse_existing_alias"
+    return _next_suffix_username(user_db, alias_base), None, "username_collision_alias"
+
+
+def upsert_external_user(
+    user_db: UserDB,
+    *,
+    auth_source: str,
+    username: str,
+    role: str,
+    email: str | None | object = UNSET,
+    display_name: str | None | object = UNSET,
+    subject_field: str | None = None,
+    subject: str | None = None,
+    allow_email_link: bool = False,
+    sync_role: bool = True,
+    allow_create: bool = True,
+    collision_strategy: CollisionStrategy = "takeover",
+    alias_suffix: str | None = None,
+    context: str | None = None,
+) -> tuple[dict[str, Any] | None, str]:
+    """Create/update a user from an external auth identity.
+
+    Returns `(user, action)` where action is one of:
+    - `"updated"`
+    - `"created"`
+    - `"not_found"` (when `allow_create=False` and no link target exists)
+    """
+    normalized_username = _normalize_username(username)
+    if not normalized_username:
+        raise ValueError("External username is required")
+
+    normalized_email = _normalize_email(email) if email is not UNSET else None
+    normalized_display_name = (
+        _normalize_display_name(display_name) if display_name is not UNSET else None
+    )
+    normalized_role = _normalize_role(role)
+
+    matched, match_reason = find_external_user_match(
+        user_db,
+        auth_source=auth_source,
+        username=normalized_username,
+        email=normalized_email,
+        subject_field=subject_field,
+        subject=subject,
+        allow_email_link=allow_email_link,
+    )
+    updates = _build_updates(
+        auth_source=auth_source,
+        role=normalized_role,
+        sync_role=sync_role,
+        email=normalized_email if email is not UNSET else UNSET,
+        display_name=normalized_display_name if display_name is not UNSET else UNSET,
+        subject_field=subject_field,
+        subject=subject,
+    )
+    if matched is not None:
+        user_db.update_user(matched["id"], **updates)
+        mapped = user_db.get_user(user_id=matched["id"]) or matched
+        logger.info(
+            "External user mapped to existing Shelfmark user "
+            f"(source={auth_source}, context={context or 'unspecified'}, reason={match_reason}, "
+            f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
+            f"shelfmark_username={mapped['username']})"
+        )
+        return mapped, "updated"
+
+    if not allow_create:
+        logger.info(
+            "External user could not be mapped and creation is disabled "
+            f"(source={auth_source}, context={context or 'unspecified'}, "
+            f"external_username={normalized_username})"
+        )
+        return None, "not_found"
+
+    resolved_alias_suffix = alias_suffix or f"__{auth_source}"
+    create_username, takeover_target, create_reason = _resolve_create_username(
+        user_db,
+        auth_source=auth_source,
+        requested_username=normalized_username,
+        strategy=collision_strategy,
+        alias_suffix=resolved_alias_suffix,
+    )
+    if takeover_target is not None:
+        user_db.update_user(takeover_target["id"], **updates)
+        mapped = user_db.get_user(user_id=takeover_target["id"]) or takeover_target
+        logger.info(
+            "External user mapped to existing Shelfmark user "
+            f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
+            f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
+            f"shelfmark_username={mapped['username']})"
+        )
+        return mapped, "updated"
+
+    create_kwargs: dict[str, Any] = {
+        "username": create_username,
+        "auth_source": auth_source,
+        "role": normalized_role,
+    }
+    if email is not UNSET:
+        create_kwargs["email"] = normalized_email
+    if display_name is not UNSET:
+        create_kwargs["display_name"] = normalized_display_name
+    if subject_field == "oidc_subject" and subject:
+        create_kwargs["oidc_subject"] = subject
+
+    created = user_db.create_user(**create_kwargs)
+    logger.info(
+        "External user created Shelfmark user "
+        f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
+        f"external_username={normalized_username}, shelfmark_user_id={created['id']}, "
+        f"shelfmark_username={created['username']})"
+    )
+    return created, "created"

shelfmark/core/image_cache.py

@@ -10,7 +10,7 @@ from typing import Any, Dict, Optional, Tuple
 
 import requests
 
-from cwa_book_downloader.core.logger import setup_logger
+from shelfmark.core.logger import setup_logger
 
 logger = setup_logger(__name__)
 
@@ -92,10 +92,13 @@ class ImageCacheService:
 
     def _load_index(self) -> None:
         """Load cache index from disk."""
+        if not self.index_path.exists():
+            self._index = {}
+            return
+
         try:
-            if self.index_path.exists():
-                with open(self.index_path, 'r') as f:
-                    self._index = json.load(f)
+            with open(self.index_path, 'r') as f:
+                self._index = json.load(f)
         except (json.JSONDecodeError, IOError):
             self._index = {}
 
@@ -179,9 +182,7 @@ class ImageCacheService:
         """Check if a cache entry is expired."""
         if self.ttl_seconds == 0:
             return False
-
-        cached_at = entry.get('cached_at', 0)
-        return (time.time() - cached_at) > self.ttl_seconds
+        return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
 
     def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
         """Check if a negative cache entry is expired.
@@ -193,12 +194,8 @@ class ImageCacheService:
             return False
 
         cached_at = entry.get('cached_at', 0)
-
-        # Transient failures (timeouts, connection errors) use shorter TTL
-        if entry.get('transient', False):
-            return (time.time() - cached_at) > TRANSIENT_CACHE_TTL
-
-        return (time.time() - cached_at) > NEGATIVE_CACHE_TTL
+        ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
+        return (time.time() - cached_at) > ttl
 
     def _calculate_total_size(self) -> int:
         """Calculate total size of cached images."""
@@ -255,14 +252,13 @@ class ImageCacheService:
         with self._lock:
             entry = self._index.get(cache_id)
 
+            # Try reloading from disk if not found (handles multiprocess case)
             if not entry:
-                # Try reloading from disk (handles multiprocess case)
                 self._load_index()
                 entry = self._index.get(cache_id)
-
-            if not entry:
-                self._misses += 1
-                return None
+                if not entry:
+                    self._misses += 1
+                    return None
 
             # Check for negative cache (failed fetch)
             if entry.get('negative', False):
@@ -526,10 +522,8 @@ class ImageCacheService:
             self.put_negative(cache_id, transient=True)
             return None
         except requests.exceptions.HTTPError as e:
-            if e.response is not None and e.response.status_code == 404:
-                self.put_negative(cache_id)
-            else:
-                self.put_negative(cache_id, transient=True)
+            is_404 = e.response is not None and e.response.status_code == 404
+            self.put_negative(cache_id, transient=not is_404)
             return None
         except Exception:
             return None
@@ -550,8 +544,8 @@ def get_image_cache() -> ImageCacheService:
     if _instance is None:
         with _instance_lock:
             if _instance is None:
-                from cwa_book_downloader.core.config import config
-                from cwa_book_downloader.config.env import CONFIG_DIR
+                from shelfmark.core.config import config
+                from shelfmark.config.env import CONFIG_DIR
 
                 cache_dir = CONFIG_DIR / "covers"
                 max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
@@ -563,7 +557,7 @@ def get_image_cache() -> ImageCacheService:
                     max_size_mb=max_size_mb,
                     ttl_seconds=ttl_seconds,
                 )
-                logger.info(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
+                logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
 
     return _instance
 

shelfmark/core/logger.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from logging.handlers import RotatingFileHandler
 from typing import Any
 
-from cwa_book_downloader.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
+from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
 
 
 class CustomLogger(logging.Logger):
@@ -39,12 +39,38 @@ class CustomLogger(logging.Logger):
         self.debug(msg, *args, exc_info=has_exception, **kwargs)
 
     def log_resource_usage(self):
-        import psutil
-        memory = psutil.virtual_memory()
-        available_mb = memory.available / (1024 * 1024)
-        memory_used_mb = memory.used / (1024 * 1024)
-        cpu_percent = psutil.cpu_percent()
-        self.debug(f"Container Memory: Available={available_mb:.2f} MB, Used={memory_used_mb:.2f} MB, CPU: {cpu_percent:.2f}%")
+        # Best-effort only; this should never raise during exception logging.
+        try:
+            import psutil
+
+            # Sum RSS of all processes for actual app memory (container-friendly),
+            # but fall back gracefully on platforms that restrict process enumeration.
+            app_memory_mb = 0.0
+            try:
+                for proc in psutil.process_iter(['memory_info']):
+                    try:
+                        mem = proc.info.get('memory_info')
+                        if mem:
+                            app_memory_mb += mem.rss / (1024 * 1024)
+                    except (psutil.NoSuchProcess, psutil.AccessDenied, KeyError, AttributeError):
+                        continue
+            except (PermissionError, psutil.AccessDenied, OSError):
+                try:
+                    app_memory_mb = psutil.Process().memory_info().rss / (1024 * 1024)
+                except Exception:
+                    app_memory_mb = 0.0
+
+            memory = psutil.virtual_memory()
+            system_used_mb = memory.used / (1024 * 1024)
+            available_mb = memory.available / (1024 * 1024)
+            cpu_percent = psutil.cpu_percent()
+            self.debug(
+                f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, "
+                f"Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%"
+            )
+        except Exception:
+            # Avoid breaking the original log call if psutil is missing or restricted.
+            return
 
 
 def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
@@ -62,17 +88,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
 
     # Create logger as CustomLogger instance
     logger = CustomLogger(name)
-    log_level = logging.INFO
-    if LOG_LEVEL == "DEBUG":
-        log_level = logging.DEBUG
-    elif LOG_LEVEL == "INFO":
-        log_level = logging.INFO
-    elif LOG_LEVEL == "WARNING":
-        log_level = logging.WARNING
-    elif LOG_LEVEL == "ERROR":
-        log_level = logging.ERROR
-    elif LOG_LEVEL == "CRITICAL":
-        log_level = logging.CRITICAL
+    log_level = getattr(logging, LOG_LEVEL, logging.INFO)
     logger.setLevel(log_level)
 
     formatter = logging.Formatter(

shelfmark/core/mirrors.py

@@ -0,0 +1,254 @@
+"""Centralized mirror configuration for all download sources."""
+
+from typing import List
+
+from shelfmark.core.utils import normalize_http_url
+
+# Lazy import to avoid circular imports
+_config_module = None
+
+
+def _get_config():
+    """Lazy import of config module to avoid circular imports."""
+    global _config_module
+    if _config_module is None:
+        from shelfmark.core.config import config
+        _config_module = config
+    return _config_module
+
+
+# Default mirror lists (hardcoded fallbacks)
+DEFAULT_AA_MIRRORS = [
+    "https://annas-archive.gl",
+    "https://annas-archive.li",
+]
+
+DEFAULT_LIBGEN_MIRRORS = [
+    "https://libgen.gl",
+    "https://libgen.li",
+    "https://libgen.bz",
+    "https://libgen.la",
+    "https://libgen.vg",
+]
+
+DEFAULT_ZLIB_MIRRORS = [
+    "https://z-lib.fm",
+    "https://z-lib.gs",
+    "https://z-lib.id",
+    "https://z-library.sk",
+    "https://zlibrary-global.se",
+]
+
+DEFAULT_WELIB_MIRRORS = [
+    "https://welib.org",
+]
+
+
+def _normalize_mirror_url(url: str) -> str:
+    return normalize_http_url(url, default_scheme="https")
+
+
+def get_aa_mirrors() -> List[str]:
+    """
+    Get Anna's Archive mirrors.
+
+    Returns:
+        Ordered list of AA mirror URLs.
+
+        If AA_MIRROR_URLS is configured, it is treated as the full list.
+        Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended.
+
+        Notes:
+        - The list is used to populate the AA mirror dropdown in Settings.
+        - When AA_BASE_URL is set to 'auto', mirrors are tried in the order listed.
+    """
+    config = _get_config()
+
+    mirrors: list[str] = []
+
+    configured_list = config.get("AA_MIRROR_URLS", None)
+    if isinstance(configured_list, list):
+        for url in configured_list:
+            normalized = _normalize_mirror_url(str(url))
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+    elif isinstance(configured_list, str) and configured_list.strip():
+        # Allow comma-separated env/manual configs.
+        for url in configured_list.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    if not mirrors:
+        mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
+        mirrors = [url for url in mirrors if url]
+
+        # Backwards-compatible append-only behavior for legacy configs/env.
+        additional = config.get("AA_ADDITIONAL_URLS", "")
+        if additional:
+            for url in additional.split(","):
+                normalized = _normalize_mirror_url(url)
+                if normalized and normalized not in mirrors:
+                    mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_libgen_mirrors() -> List[str]:
+    """
+    Get LibGen mirrors: defaults + any additional from config.
+
+    Returns:
+        List of LibGen mirror URLs (defaults first, then custom additions).
+    """
+    mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
+    mirrors = [url for url in mirrors if url]
+    config = _get_config()
+
+    additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_mirrors() -> List[str]:
+    """
+    Get Z-Library mirrors, with primary first.
+
+    Returns:
+        List of Z-Library mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_zlib_primary_url() -> str:
+    """
+    Get the primary Z-Library mirror URL.
+
+    Returns:
+        Primary Z-Library mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
+
+
+def get_zlib_url_template() -> str:
+    """
+    Get Z-Library URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_zlib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_welib_mirrors() -> List[str]:
+    """
+    Get Welib mirrors, with primary first.
+
+    Returns:
+        List of Welib mirror URLs, primary first.
+    """
+    config = _get_config()
+
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    if not primary:
+        primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+    mirrors = [primary]
+
+    # Add other defaults (excluding primary)
+    for url in DEFAULT_WELIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized and normalized != primary:
+            mirrors.append(normalized)
+
+    # Add custom mirrors
+    additional = config.get("WELIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized and normalized not in mirrors:
+                mirrors.append(normalized)
+
+    return mirrors
+
+
+def get_welib_primary_url() -> str:
+    """
+    Get the primary Welib mirror URL.
+
+    Returns:
+        Primary Welib mirror URL.
+    """
+    config = _get_config()
+    primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
+    return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
+
+
+def get_welib_url_template() -> str:
+    """
+    Get Welib URL template using configured primary mirror.
+
+    Returns:
+        URL template with {md5} placeholder.
+    """
+    primary = get_welib_primary_url()
+    return f"{primary}/md5/{{md5}}"
+
+
+def get_zlib_cookie_domains() -> set:
+    """
+    Get set of Z-Library domains that need full cookie handling.
+
+    Used by internal_bypasser for CF bypass cookie management.
+
+    Returns:
+        Set of domain strings (without protocol).
+    """
+    domains = set()
+
+    # Add all default domains
+    for url in DEFAULT_ZLIB_MIRRORS:
+        normalized = _normalize_mirror_url(url)
+        if normalized:
+            domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+            domains.add(domain)
+
+    # Add custom domains
+    config = _get_config()
+    additional = config.get("ZLIB_ADDITIONAL_URLS", "")
+    if additional:
+        for url in additional.split(","):
+            normalized = _normalize_mirror_url(url)
+            if normalized:
+                domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
+                domains.add(domain)
+
+    return domains

shelfmark/core/models.py

@@ -2,7 +2,7 @@
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from enum import Enum
 import re
 import time
@@ -14,17 +14,6 @@ def build_filename(
     year: Optional[str] = None,
     fmt: Optional[str] = None,
 ) -> str:
-    """Build sanitized filename: 'Author - Title (Year).format'
-
-    Args:
-        title: Book title (required)
-        author: Book author
-        year: Publication year
-        fmt: File format/extension
-
-    Returns:
-        Sanitized filename safe for filesystem use
-    """
     parts = []
     if author:
         parts.append(author)
@@ -46,6 +35,7 @@ class QueueStatus(str, Enum):
     """Enum for possible book queue statuses."""
     QUEUED = "queued"
     RESOLVING = "resolving"
+    LOCATING = "locating"
     DOWNLOADING = "downloading"
     COMPLETE = "complete"
     AVAILABLE = "available"
@@ -54,6 +44,11 @@ class QueueStatus(str, Enum):
     CANCELLED = "cancelled"
 
 
+class SearchMode(str, Enum):
+    DIRECT = "direct"
+    UNIVERSAL = "universal"
+
+
 @dataclass
 class QueueItem:
     """Queue item with priority and metadata."""
@@ -70,22 +65,40 @@ class QueueItem:
 
 @dataclass
 class DownloadTask:
-    """Source-agnostic download task for the queue.
-
-    This replaces BookInfo in the queue, providing a unified interface
-    for both Direct Download and Universal modes. The handler uses task_id
-    to fetch whatever source-specific data it needs internally.
-    """
     task_id: str                                # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
     source: str                                 # Handler name ("direct_download", "prowlarr")
     title: str                                  # Display title for queue sidebar
 
     # Display info for queue sidebar
     author: Optional[str] = None
+    year: Optional[str] = None
     format: Optional[str] = None
     size: Optional[str] = None
     preview: Optional[str] = None
     content_type: Optional[str] = None  # "book (fiction)", "audiobook", "magazine", etc.
+    source_url: Optional[str] = None  # Original release URL used by source-specific handlers
+
+    # Series info (for library naming templates)
+    series_name: Optional[str] = None
+    series_position: Optional[float] = None  # Float for novellas (e.g., 1.5)
+    subtitle: Optional[str] = None  # Book subtitle for naming templates
+
+    # Hardlinking support
+    original_download_path: Optional[str] = None  # Path in download client (for hardlinking)
+
+    # Search mode - determines post-download processing behavior
+    # See SearchMode enum for behavioral differences
+    search_mode: Optional[SearchMode] = None
+
+    # Output selection for post-processing.
+    # This is captured at queue time so in-flight tasks are not affected if the user changes settings later.
+    output_mode: Optional[str] = None  # e.g. "folder", "booklore", "email"
+    output_args: Dict[str, Any] = field(default_factory=dict)  # Per-output parameters (e.g. email recipient)
+
+    # User association (multi-user support)
+    user_id: Optional[int] = None  # DB user ID who queued this download
+    username: Optional[str] = None  # Username for {User} template variable
+    request_id: Optional[int] = None  # Origin request ID when queued from request fulfilment
 
     # Runtime state
     priority: int = 0
@@ -105,7 +118,7 @@ class DownloadTask:
         """Build sanitized filename from task metadata."""
         if self.download_path:
             return Path(self.download_path).name
-        return build_filename(self.title, self.author, fmt=self.format)
+        return build_filename(self.title, self.author, self.year, self.format)
 
 
 @dataclass
@@ -130,6 +143,7 @@ class BookInfo:
     status_message: Optional[str] = None  # Detailed status message for UI display
     added_time: Optional[float] = None  # Timestamp when added to queue
     source: str = "direct_download"  # Release source handler to use for downloads
+    source_url: Optional[str] = None  # Link to source page (e.g., Anna's Archive)
 
     def get_filename(self, fallback_url: Optional[str] = None) -> str:
         """Build sanitized filename: 'Author - Title (Year).format'
@@ -144,12 +158,14 @@ class BookInfo:
         """
         # Resolve format if needed
         if not self.format:
-            for url in (self.download_urls[0] if self.download_urls else None, fallback_url):
-                if url:
-                    ext = url.split(".")[-1].lower()
-                    if ext and len(ext) <= 5 and ext.isalnum():
-                        self.format = ext
-                        break
+            urls = [self.download_urls[0]] if self.download_urls else []
+            if fallback_url:
+                urls.append(fallback_url)
+            for url in urls:
+                ext = url.split(".")[-1].lower()
+                if ext and len(ext) <= 5 and ext.isalnum():
+                    self.format = ext
+                    break
 
         return build_filename(self.title, self.author, self.year, self.format)
 

shelfmark/core/naming.py

@@ -0,0 +1,248 @@
+"""Template-based naming for library organization."""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional, Union, Mapping
+
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+# Known variable tokens, sorted longest-first to avoid partial matches
+# e.g., "SeriesPosition" must match before "Series"
+KNOWN_TOKENS = [
+    'seriesposition',
+    'partnumber',
+    'subtitle',
+    'author',
+    'series',
+    'title',
+    'year',
+    'user',
+]
+
+# Match any {...} block for template parsing
+BRACE_PATTERN = re.compile(r'\{([^}]+)\}')
+
+# Characters that are invalid in filenames on various filesystems
+INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
+
+
+def _sanitize(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for filesystem use."""
+    if not name:
+        return ""
+
+    sanitized = INVALID_CHARS.sub('_', name)
+    sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized)  # Strip whitespace and dots
+    sanitized = re.sub(r'_+', '_', sanitized)  # Collapse underscores
+    return sanitized[:max_length]
+
+
+def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
+    """Sanitize a string for use as a filename or path component."""
+    return _sanitize(name, max_length)
+
+
+# Alias for backwards compatibility
+sanitize_path_component = sanitize_filename
+
+
+def format_series_position(position: Optional[Union[str, int, float]]) -> str:
+    if position is None:
+        return ""
+
+    # Display as integer if whole number
+    if isinstance(position, float) and position.is_integer():
+        return str(int(position))
+
+    return str(position)
+
+
+# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
+PAD_NUMBERS_PATTERN = re.compile(r'\d+')
+
+
+def natural_sort_key(path: Union[str, Path]) -> str:
+    """Generate a sort key with padded numbers for natural sorting."""
+    filename = Path(path).name.lower()
+    return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
+
+
+def assign_part_numbers(
+    files: list[Path],
+    zero_pad_width: int = 2,
+) -> list[tuple[Path, str]]:
+    """Sort files naturally and assign sequential part numbers (1, 2, 3...)."""
+    if not files:
+        return []
+
+    sorted_files = sorted(files, key=natural_sort_key)
+    return [
+        (file_path, str(part_num).zfill(zero_pad_width))
+        for part_num, file_path in enumerate(sorted_files, start=1)
+    ]
+
+
+def parse_naming_template(
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    *,
+    allow_path_separators: bool = True,
+) -> str:
+    if not template:
+        return ""
+
+    # Normalize metadata keys to lowercase for case-insensitive matching
+    normalized = {k.lower(): v for k, v in metadata.items()}
+
+    def find_token(content: str) -> tuple[Optional[str], int]:
+        content_lower = content.lower()
+        for token in KNOWN_TOKENS:
+            idx = content_lower.find(token)
+            if idx != -1:
+                return token, idx
+        return None, -1
+
+    def token_value(token: str) -> str:
+        value = normalized.get(token)
+        if token == 'seriesposition':
+            value = format_series_position(value)
+        if value is None:
+            return ""
+        return str(value).strip()
+
+    def render_block(content: str) -> Optional[str]:
+        token, idx = find_token(content)
+        if token is None:
+            return None
+
+        prefix = content[:idx]
+        suffix = content[idx + len(token):]
+        value = token_value(token)
+        if not value:
+            return ""
+
+        if not allow_path_separators:
+            value = value.replace("/", "_")
+        value = sanitize_filename(value)
+        return f"{prefix}{value}{suffix}"
+
+    # Process brace blocks in order so we can support conditional literal blocks like:
+    # { - Part }{PartNumber}
+    matches = list(BRACE_PATTERN.finditer(template))
+    if not matches:
+        result = template
+    else:
+        parts: list[str] = []
+        cursor = 0
+        for idx, match in enumerate(matches):
+            parts.append(template[cursor:match.start()])
+            content = match.group(1)
+            rendered = render_block(content)
+
+            if rendered is not None:
+                parts.append(rendered)
+            else:
+                conditional_literal = False
+                include_literal = False
+                if idx + 1 < len(matches) and match.end() == matches[idx + 1].start():
+                    next_content = matches[idx + 1].group(1)
+                    next_token, _next_idx = find_token(next_content)
+                    if next_token is not None:
+                        conditional_literal = True
+                        include_literal = bool(token_value(next_token))
+                if include_literal:
+                    parts.append(content)
+                elif not conditional_literal:
+                    # Preserve blocks that look like literal text, but treat bare unknown
+                    # placeholders as missing variables.
+                    if re.search(r"\s", content):
+                        parts.append(match.group(0))
+
+            cursor = match.end()
+
+        parts.append(template[cursor:])
+        result = "".join(parts)
+
+    # Clean up any double slashes that might result from empty tokens
+    result = re.sub(r'/+', '/', result)
+
+    # Remove leading/trailing slashes
+    result = result.strip('/')
+
+    # Clean up any orphaned separators (e.g., " - " at start/end, or " -  - ")
+    result = re.sub(r'^[\s\-_.]+', '', result)
+    result = re.sub(r'[\s\-_.]+$', '', result)
+    result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
+
+    # Clean up empty parentheses/brackets
+    result = re.sub(r'\(\s*\)', '', result)
+    result = re.sub(r'\[\s*\]', '', result)
+
+    # Final trim of any trailing separators left after cleanup
+    result = re.sub(r'[\s\-_.]+$', '', result)
+
+    return result
+
+
+def build_library_path(
+    base_path: str,
+    template: str,
+    metadata: Mapping[str, Optional[Union[str, int, float]]],
+    extension: Optional[str] = None,
+) -> Path:
+    relative = parse_naming_template(template, metadata, allow_path_separators=True)
+
+    if not relative:
+        # Fallback to title if template produces empty result
+        title = metadata.get('Title') or metadata.get('title') or 'Unknown'
+        relative = sanitize_filename(str(title))
+
+    # Remove any path traversal attempts
+    relative = relative.replace('..', '')
+
+    base = Path(base_path).resolve()
+    full_path = (base / relative).resolve()
+
+    # Verify the path is within the base directory
+    try:
+        full_path.relative_to(base)
+    except ValueError:
+        raise ValueError(f"Path traversal detected: template would escape library directory")
+
+    if extension:
+        ext = extension.lstrip('.')
+        # Don't use with_suffix() - it replaces everything after the first dot
+        # e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
+        full_path = Path(f"{full_path}.{ext}")
+
+    return full_path
+
+
+def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
+    """Check if two paths are on the same filesystem."""
+    path1 = Path(path1)
+    path2 = Path(path2)
+
+    def get_device(p: Path) -> Optional[int]:
+        try:
+            while not p.exists():
+                p = p.parent
+                if p == p.parent:
+                    break
+            return os.stat(p).st_dev
+        except (OSError, PermissionError) as e:
+            logger.debug(f"Cannot stat {p}: {e}")
+            return None
+
+    dev1 = get_device(path1)
+    dev2 = get_device(path2)
+
+    if dev1 is None or dev2 is None:
+        logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
+        return False
+
+    return dev1 == dev2

shelfmark/core/notifications.py

@@ -0,0 +1,380 @@
+"""Apprise notification dispatch for global and per-user events."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Iterable
+
+try:
+    import apprise
+except Exception:  # pragma: no cover - exercised in tests via monkeypatch
+    apprise = None  # type: ignore[assignment]
+
+from shelfmark.core.config import config as app_config
+from shelfmark.core.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Small pool for non-blocking dispatch. Notification sends are I/O bound and infrequent.
+_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="Notify")
+_ROUTE_EVENT_ALL = "all"
+_APPRISE_APP_ID = "Shelfmark"
+_APPRISE_APP_DESC = "Shelfmark notifications"
+_APPRISE_LOGO_URL = (
+    "https://raw.githubusercontent.com/calibrain/shelfmark/main/src/frontend/public/logo.png"
+)
+
+
+class NotificationEvent(str, Enum):
+    """Global notification event identifiers."""
+
+    REQUEST_CREATED = "request_created"
+    REQUEST_FULFILLED = "request_fulfilled"
+    REQUEST_REJECTED = "request_rejected"
+    DOWNLOAD_COMPLETE = "download_complete"
+    DOWNLOAD_FAILED = "download_failed"
+
+
+@dataclass
+class NotificationContext:
+    """Context used to render notification templates."""
+
+    event: NotificationEvent
+    title: str
+    author: str
+    username: str | None = None
+    content_type: str | None = None
+    format: str | None = None
+    source: str | None = None
+    admin_note: str | None = None
+    error_message: str | None = None
+
+
+def _normalize_urls(value: Any) -> list[str]:
+    if value is None:
+        return []
+
+    raw_values: list[Any]
+    if isinstance(value, list):
+        raw_values = value
+    elif isinstance(value, str):
+        # Support legacy/manual configs.
+        raw_values = [segment for part in value.splitlines() for segment in part.split(",")]
+    else:
+        raw_values = [value]
+
+    normalized: list[str] = []
+    seen: set[str] = set()
+    for raw_url in raw_values:
+        url = str(raw_url or "").strip()
+        if not url:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        normalized.append(url)
+    return normalized
+
+
+def _normalize_routes(value: Any) -> list[dict[str, str]]:
+    if not isinstance(value, list):
+        return []
+
+    allowed_events = {_ROUTE_EVENT_ALL, *(event.value for event in NotificationEvent)}
+    normalized: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+
+    for row in value:
+        if not isinstance(row, dict):
+            continue
+
+        raw_events = row.get("event")
+        if isinstance(raw_events, list):
+            event_values = raw_events
+        elif isinstance(raw_events, (tuple, set)):
+            event_values = list(raw_events)
+        else:
+            event_values = [raw_events]
+
+        url = str(row.get("url") or "").strip()
+        if not url:
+            continue
+
+        row_events: list[str] = []
+        for raw_event in event_values:
+            event = str(raw_event or "").strip().lower()
+            if event not in allowed_events:
+                continue
+            if event in row_events:
+                continue
+            row_events.append(event)
+
+        if _ROUTE_EVENT_ALL in row_events:
+            row_events = [_ROUTE_EVENT_ALL]
+
+        for event in row_events:
+            key = (event, url)
+            if key in seen:
+                continue
+            seen.add(key)
+
+            normalized.append({"event": event, "url": url})
+
+    return normalized
+
+
+def _resolve_admin_routes() -> list[dict[str, str]]:
+    return _normalize_routes(app_config.get("ADMIN_NOTIFICATION_ROUTES", []))
+
+
+def _normalize_user_id(value: Any) -> int | None:
+    try:
+        user_id = int(value)
+    except (TypeError, ValueError):
+        return None
+    if user_id < 1:
+        return None
+    return user_id
+
+
+def _resolve_user_routes(user_id: int | None) -> list[dict[str, str]]:
+    normalized_user_id = _normalize_user_id(user_id)
+    if normalized_user_id is None:
+        return []
+
+    return _normalize_routes(
+        app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=normalized_user_id)
+    )
+
+
+def _resolve_route_urls_for_event(
+    routes: list[dict[str, str]],
+    event: NotificationEvent,
+) -> list[str]:
+    selected: list[str] = []
+    seen: set[str] = set()
+    event_value = event.value
+
+    for row in routes:
+        row_event = row.get("event", "")
+        if row_event not in {_ROUTE_EVENT_ALL, event_value}:
+            continue
+
+        url = row.get("url", "")
+        if not url or url in seen:
+            continue
+
+        seen.add(url)
+        selected.append(url)
+
+    return selected
+
+
+def _resolve_notify_type(event: NotificationEvent) -> Any:
+    if apprise is None:
+        fallback = {
+            NotificationEvent.REQUEST_CREATED: "info",
+            NotificationEvent.REQUEST_FULFILLED: "success",
+            NotificationEvent.REQUEST_REJECTED: "warning",
+            NotificationEvent.DOWNLOAD_COMPLETE: "success",
+            NotificationEvent.DOWNLOAD_FAILED: "failure",
+        }
+        return fallback[event]
+
+    mapping = {
+        NotificationEvent.REQUEST_CREATED: apprise.NotifyType.INFO,
+        NotificationEvent.REQUEST_FULFILLED: apprise.NotifyType.SUCCESS,
+        NotificationEvent.REQUEST_REJECTED: apprise.NotifyType.WARNING,
+        NotificationEvent.DOWNLOAD_COMPLETE: apprise.NotifyType.SUCCESS,
+        NotificationEvent.DOWNLOAD_FAILED: apprise.NotifyType.FAILURE,
+    }
+    return mapping[event]
+
+
+def _clean_text(value: Any, fallback: str) -> str:
+    text = str(value or "").strip()
+    return text or fallback
+
+
+def _render_message(context: NotificationContext) -> tuple[str, str]:
+    event = context.event
+    title = _clean_text(context.title, "Unknown title")
+    author = _clean_text(context.author, "Unknown author")
+    username = _clean_text(context.username, "A user")
+
+    if event == NotificationEvent.REQUEST_CREATED:
+        return "New Request", f'{username} requested "{title}" by {author}'
+    if event == NotificationEvent.REQUEST_FULFILLED:
+        return "Request Approved", f'Request for "{title}" by {author} was approved.'
+    if event == NotificationEvent.REQUEST_REJECTED:
+        note = _clean_text(context.admin_note, "")
+        note_line = f"\nNote: {note}" if note else ""
+        return "Request Rejected", f'Request for "{title}" by {author} was rejected.{note_line}'
+    if event == NotificationEvent.DOWNLOAD_COMPLETE:
+        return "Download Complete", f'"{title}" by {author} downloaded successfully.'
+
+    error_message = _clean_text(context.error_message, "")
+    error_line = f"\nError: {error_message}" if error_message else ""
+    return "Download Failed", f'Failed to download "{title}" by {author}.{error_line}'
+
+
+def _dispatch_to_apprise(
+    urls: Iterable[str],
+    *,
+    title: str,
+    body: str,
+    notify_type: Any,
+) -> dict[str, Any]:
+    normalized_urls = _normalize_urls(list(urls))
+    if not normalized_urls:
+        return {"success": False, "message": "No notification URLs configured"}
+
+    if apprise is None:
+        return {"success": False, "message": "Apprise is not installed"}
+
+    apobj = _create_apprise_client()
+    if apobj is None:
+        return {"success": False, "message": "Apprise is not installed"}
+    valid_urls = 0
+    invalid_urls = 0
+    for url in normalized_urls:
+        try:
+            added = bool(apobj.add(url))
+        except Exception:
+            added = False
+        if added:
+            valid_urls += 1
+        else:
+            invalid_urls += 1
+
+    if valid_urls == 0:
+        return {
+            "success": False,
+            "message": "No valid notification URLs configured",
+        }
+
+    try:
+        delivered = bool(apobj.notify(title=title, body=body, notify_type=notify_type))
+    except Exception as exc:
+        return {"success": False, "message": f"Notification send failed: {type(exc).__name__}: {exc}"}
+
+    if not delivered:
+        return {"success": False, "message": "Notification delivery failed"}
+
+    message = f"Notification sent to {valid_urls} URL(s)"
+    if invalid_urls:
+        message += f" ({invalid_urls} invalid URL(s) skipped)"
+    return {"success": True, "message": message}
+
+
+def _create_apprise_client() -> Any:
+    if apprise is None:
+        return None
+
+    apprise_cls = getattr(apprise, "Apprise", None)
+    if apprise_cls is None:
+        return None
+
+    apprise_asset_cls = getattr(apprise, "AppriseAsset", None)
+    if apprise_asset_cls is None:
+        return apprise_cls()
+
+    try:
+        asset = apprise_asset_cls(
+            app_id=_APPRISE_APP_ID,
+            app_desc=_APPRISE_APP_DESC,
+            image_url_logo=_APPRISE_LOGO_URL,
+        )
+    except TypeError:
+        # Support older Apprise versions that do not expose image_url_logo.
+        asset = apprise_asset_cls(
+            app_id=_APPRISE_APP_ID,
+            app_desc=_APPRISE_APP_DESC,
+        )
+    except Exception:
+        return apprise_cls()
+
+    try:
+        return apprise_cls(asset=asset)
+    except Exception:
+        return apprise_cls()
+
+
+def _send_admin_event(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> dict[str, Any]:
+    title, body = _render_message(context)
+    notify_type = _resolve_notify_type(event)
+    return _dispatch_to_apprise(urls, title=title, body=body, notify_type=notify_type)
+
+
+def notify_admin(event: NotificationEvent, context: NotificationContext) -> None:
+    """Send a global admin notification for an event if subscribed."""
+    routes = _resolve_admin_routes()
+    urls = _resolve_route_urls_for_event(routes, event)
+    if not urls:
+        return
+
+    try:
+        _executor.submit(_dispatch_admin_async, event, context, urls)
+    except Exception as exc:
+        logger.warning("Failed to queue admin notification '%s': %s", event.value, exc)
+
+
+def notify_user(user_id: int | None, event: NotificationEvent, context: NotificationContext) -> None:
+    """Send a per-user notification for an event if subscribed."""
+    normalized_user_id = _normalize_user_id(user_id)
+    if normalized_user_id is None:
+        return
+
+    routes = _resolve_user_routes(normalized_user_id)
+    urls = _resolve_route_urls_for_event(routes, event)
+    if not urls:
+        return
+
+    try:
+        _executor.submit(_dispatch_user_async, normalized_user_id, event, context, urls)
+    except Exception as exc:
+        logger.warning(
+            "Failed to queue user notification '%s' for user_id=%s: %s",
+            event.value,
+            normalized_user_id,
+            exc,
+        )
+
+
+def _dispatch_admin_async(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> None:
+    result = _send_admin_event(event, context, urls)
+    if not result.get("success", False):
+        logger.warning("Admin notification failed for event '%s': %s", event.value, result.get("message"))
+
+
+def _dispatch_user_async(
+    user_id: int,
+    event: NotificationEvent,
+    context: NotificationContext,
+    urls: list[str],
+) -> None:
+    result = _send_admin_event(event, context, urls)
+    if not result.get("success", False):
+        logger.warning(
+            "User notification failed for event '%s' (user_id=%s): %s",
+            event.value,
+            user_id,
+            result.get("message"),
+        )
+
+
+def send_test_notification(urls: list[str]) -> dict[str, Any]:
+    """Send a synchronous test notification to the provided URLs."""
+    normalized_urls = _normalize_urls(urls)
+    if not normalized_urls:
+        return {"success": False, "message": "No notification URLs configured"}
+
+    test_context = NotificationContext(
+        event=NotificationEvent.REQUEST_CREATED,
+        title="Shelfmark Test Notification",
+        author="Shelfmark",
+        username="Shelfmark",
+    )
+    return _send_admin_event(NotificationEvent.REQUEST_CREATED, test_context, normalized_urls)

shelfmark/core/oidc_auth.py

@@ -0,0 +1,80 @@
+"""OIDC authentication helpers.
+
+Handles group claim parsing, user info extraction, and user provisioning.
+Flask route handlers are registered separately in main.py.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.external_user_linking import upsert_external_user
+from shelfmark.core.user_db import UserDB
+
+def parse_group_claims(id_token: Dict[str, Any], group_claim: str) -> List[str]:
+    """Extract group list from an ID token claim.
+
+    Supports list, comma-separated string, or pipe-separated string.
+    Returns empty list if claim is missing.
+    """
+    raw = id_token.get(group_claim)
+    if raw is None:
+        return []
+    if isinstance(raw, list):
+        return [str(g).strip() for g in raw if str(g).strip()]
+    if isinstance(raw, str):
+        delimiter = "," if "," in raw else "|"
+        return [g.strip() for g in raw.split(delimiter) if g.strip()]
+    return []
+
+
+def extract_user_info(id_token: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract user info from OIDC ID token claims.
+
+    Returns a dict with keys: oidc_subject, username, email, display_name.
+    Falls back through preferred_username -> email -> sub for username.
+    """
+    sub = id_token.get("sub", "")
+    email = id_token.get("email")
+    display_name = id_token.get("name")
+    username = id_token.get("preferred_username") or email or sub
+
+    return {
+        "oidc_subject": sub,
+        "username": username,
+        "email": email,
+        "display_name": display_name,
+    }
+
+
+def provision_oidc_user(
+    db: UserDB,
+    user_info: Dict[str, Any],
+    is_admin: Optional[bool] = None,
+    allow_email_link: bool = False,
+    allow_create: bool = True,
+) -> Optional[Dict[str, Any]]:
+    """Create or update a user from OIDC claims.
+
+    Matching and collision handling use the shared external user linker:
+    - OIDC subject first
+    - optionally unique email linking (when `allow_email_link=True`)
+    - username conflict resolution via numeric suffix.
+
+    Returns None when no existing user is matchable and `allow_create=False`.
+    """
+    oidc_subject = user_info["oidc_subject"]
+    user, _ = upsert_external_user(
+        db,
+        auth_source="oidc",
+        username=user_info["username"] or oidc_subject,
+        role="admin" if is_admin else "user",
+        email=user_info.get("email"),
+        display_name=user_info.get("display_name"),
+        subject_field="oidc_subject",
+        subject=oidc_subject,
+        allow_email_link=allow_email_link,
+        sync_role=is_admin is not None,
+        allow_create=allow_create,
+        collision_strategy="suffix",
+        context="oidc_login",
+    )
+    return user

shelfmark/core/oidc_routes.py

@@ -0,0 +1,210 @@
+"""OIDC Flask route handlers using Authlib.
+
+Registers /api/auth/oidc/login and /api/auth/oidc/callback endpoints.
+Business logic remains in oidc_auth.py.
+"""
+
+from typing import Any
+
+from authlib.jose.errors import InvalidClaimError
+from authlib.integrations.flask_client import OAuth
+from flask import Flask, jsonify, redirect, request, session
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.oidc_auth import (
+    extract_user_info,
+    parse_group_claims,
+    provision_oidc_user,
+)
+from shelfmark.core.settings_registry import load_config_file
+from shelfmark.core.user_db import UserDB
+
+logger = setup_logger(__name__)
+oauth = OAuth()
+
+
+def _normalize_claims(raw_claims: Any) -> dict[str, Any]:
+    """Return a plain dict for claims from Authlib token/userinfo payloads."""
+    if raw_claims is None:
+        return {}
+    if isinstance(raw_claims, dict):
+        return raw_claims
+    if hasattr(raw_claims, "to_dict"):
+        return raw_claims.to_dict()  # type: ignore[no-any-return]
+    try:
+        return dict(raw_claims)
+    except Exception:
+        return {}
+
+
+def _is_email_verified(claims: dict[str, Any]) -> bool:
+    """Normalize provider-specific email_verified values into a strict boolean."""
+    value = claims.get("email_verified", False)
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() == "true"
+    return False
+
+
+def _get_oidc_client() -> tuple[Any, dict[str, Any]]:
+    """Register and return an OIDC client from the current security config."""
+    config = load_config_file("security")
+    discovery_url = config.get("OIDC_DISCOVERY_URL", "")
+    client_id = config.get("OIDC_CLIENT_ID", "")
+
+    if not discovery_url or not client_id:
+        raise ValueError("OIDC not configured")
+
+    configured_scopes = config.get("OIDC_SCOPES", ["openid", "email", "profile"])
+    if isinstance(configured_scopes, list):
+        scope_values = [str(scope).strip() for scope in configured_scopes if str(scope).strip()]
+    elif isinstance(configured_scopes, str):
+        delimiter = "," if "," in configured_scopes else " "
+        scope_values = [scope.strip() for scope in configured_scopes.split(delimiter) if scope.strip()]
+    else:
+        scope_values = []
+
+    scopes = list(dict.fromkeys(["openid"] + scope_values))
+
+    admin_group = config.get("OIDC_ADMIN_GROUP", "")
+    group_claim = config.get("OIDC_GROUP_CLAIM", "groups")
+    use_admin_group = config.get("OIDC_USE_ADMIN_GROUP", True)
+    if admin_group and use_admin_group and group_claim and group_claim not in scopes:
+        scopes.append(group_claim)
+
+    oauth._clients.pop("shelfmark_idp", None)
+    oauth.register(
+        name="shelfmark_idp",
+        client_id=client_id,
+        client_secret=config.get("OIDC_CLIENT_SECRET", ""),
+        server_metadata_url=discovery_url,
+        client_kwargs={
+            "scope": " ".join(scopes),
+            "code_challenge_method": "S256",
+        },
+        overwrite=True,
+    )
+
+    client = oauth.create_client("shelfmark_idp")
+    if client is None:
+        raise RuntimeError("OIDC client initialization failed")
+
+    return client, config
+
+
+def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
+    """Register OIDC authentication routes on the Flask app."""
+    oauth.init_app(app)
+
+    @app.route("/api/auth/oidc/login", methods=["GET"])
+    def oidc_login():
+        """Initiate OIDC login flow and redirect to the provider."""
+        try:
+            client, _ = _get_oidc_client()
+            redirect_uri = request.url_root.rstrip("/") + "/api/auth/oidc/callback"
+            return client.authorize_redirect(redirect_uri)
+        except ValueError:
+            return jsonify({"error": "OIDC not configured"}), 500
+        except Exception as e:
+            logger.error(f"OIDC login error: {e}")
+            return jsonify({"error": "OIDC login failed"}), 500
+
+    @app.route("/api/auth/oidc/callback", methods=["GET"])
+    def oidc_callback():
+        """Handle OIDC callback from identity provider."""
+        try:
+            error = request.args.get("error")
+            if error:
+                logger.warning(f"OIDC callback error from IdP: {error}")
+                return jsonify({"error": "Authentication failed"}), 400
+
+            client, config = _get_oidc_client()
+            try:
+                token = client.authorize_access_token()
+            except InvalidClaimError as e:
+                claim_name = getattr(e, "claim_name", "unknown")
+                discovery_url = str(config.get("OIDC_DISCOVERY_URL", ""))
+                provider_issuer = ""
+                try:
+                    metadata = client.load_server_metadata()
+                    if isinstance(metadata, dict):
+                        provider_issuer = str(metadata.get("issuer", ""))
+                except Exception as metadata_error:
+                    logger.debug(f"OIDC metadata lookup failed during claim diagnostics: {metadata_error}")
+
+                logger.error(
+                    "OIDC callback claim validation failed: claim=%s error=%s discovery_url=%s provider_issuer=%s",
+                    claim_name,
+                    e,
+                    discovery_url or "<unset>",
+                    provider_issuer or "<unknown>",
+                )
+                if claim_name == "iss":
+                    return (
+                        jsonify(
+                            {
+                                "error": (
+                                    "OIDC issuer validation failed. Verify your discovery URL and IdP issuer/"
+                                    "external URL configuration."
+                                )
+                            }
+                        ),
+                        400,
+                    )
+
+                return jsonify({"error": f"OIDC token claim validation failed: {claim_name}"}), 400
+            claims = _normalize_claims(token.get("userinfo"))
+
+            # If userinfo isn't present in token payload, request it explicitly.
+            if not claims:
+                try:
+                    claims = _normalize_claims(client.userinfo(token=token))
+                except TypeError:
+                    claims = _normalize_claims(client.userinfo())
+                except Exception as e:
+                    logger.error(f"Failed to fetch OIDC userinfo: {e}")
+
+            if not claims:
+                raise ValueError("OIDC authentication failed: missing user claims")
+
+            group_claim = config.get("OIDC_GROUP_CLAIM", "groups")
+            admin_group = config.get("OIDC_ADMIN_GROUP", "")
+            use_admin_group = config.get("OIDC_USE_ADMIN_GROUP", True)
+            auto_provision = config.get("OIDC_AUTO_PROVISION", True)
+
+            user_info = extract_user_info(claims)
+            groups = parse_group_claims(claims, group_claim)
+
+            is_admin = None
+            if admin_group and use_admin_group:
+                is_admin = admin_group in groups
+
+            allow_email_link = bool(user_info.get("email")) and _is_email_verified(claims)
+            user = provision_oidc_user(
+                user_db,
+                user_info,
+                is_admin=is_admin,
+                allow_email_link=allow_email_link,
+                allow_create=bool(auto_provision),
+            )
+            if user is None:
+                logger.warning(
+                    f"OIDC login rejected: auto-provision disabled for {user_info['username']}"
+                )
+                return jsonify({"error": "Account not found. Contact your administrator."}), 403
+
+            session["user_id"] = user["username"]
+            session["is_admin"] = user.get("role") == "admin"
+            session["db_user_id"] = user["id"]
+            session.permanent = True
+
+            logger.info(f"OIDC login successful: {user['username']} (admin={is_admin})")
+            return redirect(request.script_root or "/")
+
+        except ValueError as e:
+            logger.error(f"OIDC callback error: {e}")
+            return jsonify({"error": str(e)}), 400
+        except Exception as e:
+            logger.error(f"OIDC callback error: {e}")
+            return jsonify({"error": "Authentication failed"}), 500

shelfmark/core/onboarding.py

@@ -0,0 +1,435 @@
+"""
+Onboarding wizard configuration.
+
+Defines the steps and fields for the first-run onboarding experience.
+Reuses field definitions from the settings registry where possible.
+"""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from shelfmark.core.logger import setup_logger
+from shelfmark.core.settings_registry import (
+    HeadingField,
+    SettingsField,
+    get_settings_tab,
+    serialize_field,
+    save_config_file,
+    get_setting_value,
+)
+
+logger = setup_logger(__name__)
+
+
+ONBOARDING_STORAGE_KEY = "onboarding_complete"
+
+
+def _get_config_dir() -> Path:
+    """Get the config directory path."""
+    from shelfmark.config.env import CONFIG_DIR
+    return Path(CONFIG_DIR)
+
+
+def is_onboarding_complete() -> bool:
+    """Check if onboarding has been completed."""
+    from shelfmark.config.env import ONBOARDING
+
+    # If onboarding is disabled via env var, treat as complete
+    if not ONBOARDING:
+        return True
+
+    config_file = _get_config_dir() / "settings.json"
+    if not config_file.exists():
+        return False
+
+    try:
+        with open(config_file, 'r') as f:
+            config = json.load(f)
+            return config.get(ONBOARDING_STORAGE_KEY, False)
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning(f"Could not read onboarding status from settings.json: {e}")
+        return False
+
+
+def mark_onboarding_complete() -> bool:
+    """Mark onboarding as complete."""
+    try:
+        return save_config_file("general", {ONBOARDING_STORAGE_KEY: True})
+    except Exception as e:
+        logger.error(f"Failed to mark onboarding complete: {e}")
+        return False
+
+
+def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField]:
+    """
+    Extract a specific field from a registered settings tab.
+
+    Args:
+        tab_name: Name of the settings tab (e.g., 'search_mode', 'hardcover')
+        field_key: Key of the field to extract (e.g., 'SEARCH_MODE', 'HARDCOVER_API_KEY')
+
+    Returns:
+        The field if found, None otherwise
+    """
+    tab = get_settings_tab(tab_name)
+    if not tab:
+        logger.warning(f"Settings tab not found: {tab_name}")
+        return None
+
+    for field in tab.fields:
+        if hasattr(field, 'key') and field.key == field_key:
+            return field
+
+    logger.warning(f"Field {field_key} not found in tab {tab_name}")
+    return None
+
+
+def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
+    """
+    Clone a field with optional attribute overrides.
+
+    Useful for customizing labels, descriptions, or defaults for onboarding context.
+    """
+    return replace(field, **overrides)
+
+
+# =============================================================================
+# Step Definitions
+# =============================================================================
+
+
+def get_search_mode_fields() -> List[SettingsField]:
+    """Step 1: Choose search mode - uses actual SEARCH_MODE field from settings."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="welcome_heading",
+            title="Welcome to Shelfmark",
+            description="Let's configure how you want to search for and download books.",
+        ),
+    ]
+
+    # Get the actual SEARCH_MODE field from settings
+    search_mode_field = _get_field_from_tab("search_mode", "SEARCH_MODE")
+    if search_mode_field:
+        # Clone with onboarding-specific description
+        fields.append(_clone_field_with_overrides(
+            search_mode_field,
+            description="Choose how you want to find books.",
+        ))
+
+    return fields
+
+
+def get_metadata_provider_fields() -> List[SettingsField]:
+    """Step 2: Choose metadata provider - uses actual METADATA_PROVIDER field."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="metadata_heading",
+            title="Metadata Provider",
+            description="Choose where to search for book information. You can enable more providers in Settings later.",
+        ),
+    ]
+
+    # Get the actual METADATA_PROVIDER field from settings
+    provider_field = _get_field_from_tab("search_mode", "METADATA_PROVIDER")
+    if provider_field:
+        # Custom options with Hardcover marked as recommended
+        onboarding_options = [
+            {
+                "value": "hardcover",
+                "label": "Hardcover (Recommended)",
+                "description": "Modern book tracking platform with excellent metadata, ratings, and series information. Requires free API key.",
+            },
+            {
+                "value": "openlibrary",
+                "label": "Open Library",
+                "description": "Free, open-source library catalog from the Internet Archive. No API key required.",
+            },
+            {
+                "value": "googlebooks",
+                "label": "Google Books",
+                "description": "Google's book database with good coverage. Requires free API key.",
+            },
+        ]
+
+        # Clone with onboarding-specific options and default
+        fields.append(_clone_field_with_overrides(
+            provider_field,
+            default="hardcover",
+            options=onboarding_options,
+        ))
+
+    return fields
+
+
+def get_hardcover_setup_fields() -> List[SettingsField]:
+    """Step 3a: Configure Hardcover - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="hardcover_setup_heading",
+            title="Hardcover Setup",
+            description="Get your free API key from hardcover.app/account/api",
+            link_url="https://hardcover.app/account/api",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual HARDCOVER_API_KEY field
+    api_key_field = _get_field_from_tab("hardcover", "HARDCOVER_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("hardcover", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_googlebooks_setup_fields() -> List[SettingsField]:
+    """Step 3b: Configure Google Books - uses actual API key and test connection fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="googlebooks_setup_heading",
+            title="Google Books Setup",
+            description="Get your free API key from Google Cloud Console (APIs & Services > Credentials).",
+            link_url="https://console.cloud.google.com/apis/library/books.googleapis.com",
+            link_text="Get API Key",
+        ),
+    ]
+
+    # Get the actual GOOGLEBOOKS_API_KEY field
+    api_key_field = _get_field_from_tab("googlebooks", "GOOGLEBOOKS_API_KEY")
+    if api_key_field:
+        fields.append(api_key_field)
+
+    # Get the test connection button
+    test_button = _get_field_from_tab("googlebooks", "test_connection")
+    if test_button:
+        fields.append(test_button)
+
+    return fields
+
+
+def get_prowlarr_fields() -> List[SettingsField]:
+    """Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_heading",
+            title="Prowlarr Integration (Optional)",
+            description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.",
+        ),
+    ]
+
+    # Get actual Prowlarr connection fields
+    prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"]
+    for field_key in prowlarr_fields:
+        field = _get_field_from_tab("prowlarr_config", field_key)
+        if field:
+            fields.append(field)
+
+    return fields
+
+
+def get_prowlarr_indexers_fields() -> List[SettingsField]:
+    """Step 5: Select Prowlarr indexers to search."""
+    fields: List[SettingsField] = [
+        HeadingField(
+            key="prowlarr_indexers_heading",
+            title="Select Indexers",
+            description="Choose which indexers to search for books. Leave empty to search all available indexers.",
+        ),
+    ]
+
+    # Get the indexers multi-select field
+    indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS")
+    if indexers_field:
+        fields.append(indexers_field)
+
+    return fields
+
+
+# =============================================================================
+# Step Configuration
+# =============================================================================
+
+
+ONBOARDING_STEPS = [
+    {
+        "id": "search_mode",
+        "title": "Search Mode",
+        "tab": "search_mode",
+        "get_fields": get_search_mode_fields,
+    },
+    {
+        "id": "metadata_provider",
+        "title": "Metadata Provider",
+        "tab": "search_mode",
+        "get_fields": get_metadata_provider_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+    },
+    {
+        "id": "hardcover_setup",
+        "title": "Hardcover Setup",
+        "tab": "hardcover",
+        "get_fields": get_hardcover_setup_fields,
+        # Must be universal mode AND hardcover selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "hardcover"},
+        ],
+    },
+    {
+        "id": "googlebooks_setup",
+        "title": "Google Books Setup",
+        "tab": "googlebooks",
+        "get_fields": get_googlebooks_setup_fields,
+        # Must be universal mode AND googlebooks selected
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "METADATA_PROVIDER", "value": "googlebooks"},
+        ],
+    },
+    {
+        "id": "prowlarr",
+        "title": "Prowlarr",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_fields,
+        "show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
+        "optional": True,
+    },
+    {
+        "id": "prowlarr_indexers",
+        "title": "Indexers",
+        "tab": "prowlarr_config",
+        "get_fields": get_prowlarr_indexers_fields,
+        # Only show when Prowlarr is enabled
+        "show_when": [
+            {"field": "SEARCH_MODE", "value": "universal"},
+            {"field": "PROWLARR_ENABLED", "value": True},
+        ],
+        "optional": True,
+    },
+]
+
+
+def get_onboarding_config() -> Dict[str, Any]:
+    """
+    Get the full onboarding configuration including steps and current values.
+    """
+    steps = []
+    all_values = {}
+
+    for step_config in ONBOARDING_STEPS:
+        fields = step_config["get_fields"]()
+        tab_name = step_config["tab"]
+
+        # Serialize fields with current values
+        serialized_fields = []
+        for field in fields:
+            serialized = serialize_field(field, tab_name, include_value=True)
+            serialized_fields.append(serialized)
+
+            # Collect values (skip HeadingFields)
+            if hasattr(field, 'key') and field.key and not isinstance(field, HeadingField):
+                value = get_setting_value(field, tab_name)
+                all_values[field.key] = value if value is not None else getattr(field, 'default', '')
+
+        step = {
+            "id": step_config["id"],
+            "title": step_config["title"],
+            "tab": tab_name,
+            "fields": serialized_fields,
+        }
+
+        if "show_when" in step_config:
+            step["showWhen"] = step_config["show_when"]
+        if step_config.get("optional"):
+            step["optional"] = True
+
+        steps.append(step)
+
+    return {
+        "steps": steps,
+        "values": all_values,
+        "complete": is_onboarding_complete(),
+    }
+
+
+def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Save onboarding settings and mark as complete.
+
+    Args:
+        values: Dict of field key -> value
+
+    Returns:
+        Dict with success status and message
+    """
+    try:
+        # Group values by their target tab
+        tab_values: Dict[str, Dict[str, Any]] = {}
+
+        for step_config in ONBOARDING_STEPS:
+            tab_name = step_config["tab"]
+            fields = step_config["get_fields"]()
+
+            for field in fields:
+                if isinstance(field, HeadingField):
+                    continue
+
+                key = field.key
+                if key in values:
+                    if tab_name not in tab_values:
+                        tab_values[tab_name] = {}
+                    tab_values[tab_name][key] = values[key]
+
+        # Save each tab's values
+        for tab_name, tab_data in tab_values.items():
+            if tab_data:
+                save_config_file(tab_name, tab_data)
+                logger.info(f"Saved onboarding settings to {tab_name}: {list(tab_data.keys())}")
+
+        # Enable the selected metadata provider
+        search_mode = values.get("SEARCH_MODE", "direct")
+        if search_mode == "universal":
+            provider = values.get("METADATA_PROVIDER", "hardcover")
+            if provider:
+                # Map provider name to its enabled key
+                enabled_key_map = {
+                    "hardcover": "HARDCOVER_ENABLED",
+                    "openlibrary": "OPENLIBRARY_ENABLED",
+                    "googlebooks": "GOOGLEBOOKS_ENABLED",
+                }
+                enabled_key = enabled_key_map.get(provider, f"{provider.upper()}_ENABLED")
+
+                # Get existing provider config and add enabled flag
+                provider_config = {enabled_key: True}
+
+                # Include API key if provided for that provider
+                if provider == "hardcover" and values.get("HARDCOVER_API_KEY"):
+                    provider_config["HARDCOVER_API_KEY"] = values["HARDCOVER_API_KEY"]
+                elif provider == "googlebooks" and values.get("GOOGLEBOOKS_API_KEY"):
+                    provider_config["GOOGLEBOOKS_API_KEY"] = values["GOOGLEBOOKS_API_KEY"]
+
+                save_config_file(provider, provider_config)
+                logger.info(f"Enabled metadata provider: {provider} with keys: {list(provider_config.keys())}")
+
+        # Mark onboarding as complete
+        mark_onboarding_complete()
+
+        # Refresh config
+        try:
+            from shelfmark.core.config import config
+            config.refresh()
+        except ImportError as e:
+            logger.debug(f"Could not refresh config after onboarding: {e}")
+
+        return {"success": True, "message": "Onboarding complete!"}
+
+    except Exception as e:
+        logger.error(f"Failed to save onboarding settings: {e}")
+        return {"success": False, "message": str(e)}